[Refactor] remove the c implementation of flow_warp (#707)

* remove flow-warp-c

* remove flow warp in setup.py and imports

* fix floor and ceil bug

* fix broadcast bug

* add warnings and assertions

* fix bilinear bug

* pass unit test

* pass unit test

* pass unit test

* pass unit test

* fix value bug

* fix lint

* fix lint

* add mk lint

* update docs

* update docs

* fix bug in imports

* fix bug in setup.py

.gitignore

@@ -109,4 +109,3 @@ venv.bak/
 
 # custom
 .DS_Store
-mmcv/video/optflow_warp/flow_warp_module.cpp

MANIFEST.in

-include mmcv/video/optflow_warp/*.hpp mmcv/video/optflow_warp/*.pyx
 include requirements/runtime.txt
 include mmcv/model_zoo/open_mmlab.json mmcv/model_zoo/deprecated.json mmcv/model_zoo/mmcls.json
 include mmcv/ops/csrc/*.cuh mmcv/ops/csrc/*.hpp

docs/build.md

@@ -101,11 +101,11 @@ MMCV can be built in three ways:
 
 1. Lite version (without ops)
 
-   In this way, only `flow_warp` module will be compiled as a Cython extension.
+   In this way, no custom ops are compiled and mmcv is a pure python package.
 
 1. Full version (CPU ops)
 
-   In addition to `flow_warp` module, module `ops` will be compiled as a pytorch extension, but only x86 code will be compiled. The compiled ops can be executed on CPU only.
+   Module `ops` will be compiled as a pytorch extension, but only x86 code will be compiled. The compiled ops can be executed on CPU only.
 
 1. Full version (CUDA ops)
 
@@ -138,9 +138,6 @@ After finishing above common steps, launch Anaconda shell from Start menu and is
 conda activate mmcv
 # change directory
 cd mmcv
-# build
-$env:MMCV_WITH_OPS = 0
-python setup.py build_ext # if success, cl will be launched to compile flow_warp.
 # install
 python setup.py develop
 # check
@@ -165,7 +162,7 @@ pip list
     # change directory
     cd mmcv
     # build
-    python setup.py build_ext # if success, cl will be launched to compile flow_warp first and then ops
+    python setup.py build_ext # if success, cl will be launched to compile ops
     # install
     python setup.py develop
     # check
@@ -218,7 +215,7 @@ pip list
    # change directory
    cd mmcv
    # build
-   python setup.py build_ext # if success, cl will be launched to compile flow_warp first and then ops
+   python setup.py build_ext # if success, cl will be launched to compile ops
    # install
    python setup.py develop
    # check

docs/conf.py

@@ -53,8 +53,7 @@ extensions = [
 ]
 
 autodoc_mock_imports = [
-    'cv2', 'mmcv._ext', 'mmcv._flow_warp_ext', 'mmcv.utils.ext_loader',
-    'torchvision'
+    'cv2', 'mmcv._ext', 'mmcv.utils.ext_loader', 'torchvision'
 ]
 autosectionlabel_prefix_document = True
 

mmcv/video/optflow.py

 # Copyright (c) Open-MMLab. All rights reserved.
+import warnings
+
 import numpy as np
 
-from mmcv._flow_warp_ext import flow_warp_c
 from mmcv.arraymisc import dequantize, quantize
 from mmcv.image import imread, imwrite
 from mmcv.utils import is_str
@@ -151,19 +152,49 @@ def flow_warp(img, flow, filling_value=0, interpolate_mode='nearest'):
     Returns:
         ndarray: Warped image with the same shape of img
     """
-    interpolate_mode_dict = {'bilinear': 0, 'nearest': 1}
-    assert len(img.shape) == 3
-    assert len(flow.shape) == 3 and flow.shape[2] == 2
-    assert flow.shape[:2] == img.shape[:2]
-    assert interpolate_mode in interpolate_mode_dict.keys()
-
-    interpolate_mode = interpolate_mode_dict[interpolate_mode]
-    img_float = img.astype(np.float64)
-
-    out = flow_warp_c(
-        img_float,
-        flow.astype(np.float64),
-        filling_value=filling_value,
-        interpolate_mode=interpolate_mode)
-
-    return out
+    warnings.warn('This function is just for prototyping and cannot '
+                  'guarantee the computational efficiency.')
+    assert flow.ndim == 3, 'Flow must be in 3D arrays.'
+    height = flow.shape[0]
+    width = flow.shape[1]
+    channels = img.shape[2]
+
+    output = np.ones(
+        (height, width, channels), dtype=img.dtype) * filling_value
+
+    grid = np.indices((height, width)).swapaxes(0, 1).swapaxes(1, 2)
+    dx = grid[:, :, 0] + flow[:, :, 1]
+    dy = grid[:, :, 1] + flow[:, :, 0]
+    sx = np.floor(dx).astype(int)
+    sy = np.floor(dy).astype(int)
+    valid = (sx >= 0) & (sx < height - 1) & (sy >= 0) & (sy < width - 1)
+
+    if interpolate_mode == 'nearest':
+        output[valid, :] = img[dx[valid].round().astype(int),
+                               dy[valid].round().astype(int), :]
+    elif interpolate_mode == 'bilinear':
+        # dirty walkround for integer positions
+        eps_ = 1e-6
+        dx, dy = dx + eps_, dy + eps_
+        left_top_ = img[np.floor(dx[valid]).astype(int),
+                        np.floor(dy[valid]).astype(int), :] * (
+                            np.ceil(dx[valid]) - dx[valid])[:, None] * (
+                                np.ceil(dy[valid]) - dy[valid])[:, None]
+        left_down_ = img[np.ceil(dx[valid]).astype(int),
+                         np.floor(dy[valid]).astype(int), :] * (
+                             dx[valid] - np.floor(dx[valid]))[:, None] * (
+                                 np.ceil(dy[valid]) - dy[valid])[:, None]
+        right_top_ = img[np.floor(dx[valid]).astype(int),
+                         np.ceil(dy[valid]).astype(int), :] * (
+                             np.ceil(dx[valid]) - dx[valid])[:, None] * (
+                                 dy[valid] - np.floor(dy[valid]))[:, None]
+        right_down_ = img[np.ceil(dx[valid]).astype(int),
+                          np.ceil(dy[valid]).astype(int), :] * (
+                              dx[valid] - np.floor(dx[valid]))[:, None] * (
+                                  dy[valid] - np.floor(dy[valid]))[:, None]
+        output[valid, :] = left_top_ + left_down_ + right_top_ + right_down_
+    else:
+        raise NotImplementedError(
+            'We only support interpolation modes of nearest and bilinear, '
+            f'but got {interpolate_mode}.')
+    return output.astype(img.dtype)

mmcv/video/optflow_warp/__init__.py

-# Copyright (c) Open-MMLab. All rights reserved.

mmcv/video/optflow_warp/flow_warp.cpp

-// Copyright (c) Open-MMLab. All rights reserved.
-#include "flow_warp.hpp"
-
-void FlowWarp(double* img, double* flow, double* out, const int height,
-              const int width, const int channels, const int filling_value = 0,
-              const int interpolateMode = 0) {
-  for (int h = 0; h < height; h++) {
-    for (int w = 0; w < width; w++) {
-      int offset_cur = h * width + w;
-      int offset_img = offset_cur * channels;
-      int offset_flow = offset_cur * 2;
-      double x, y;
-      x = h + flow[offset_flow + 1];
-      y = w + flow[offset_flow];
-
-      if (x < 0 || x >= height - 1 || y < 0 || y >= width - 1) {
-        for (int k = 0; k < channels; k++) {
-          out[offset_img + k] = filling_value;
-        }
-        continue;
-      }
-
-      if (interpolateMode == 0)
-        BilinearInterpolate(img, width, height, channels, x, y,
-                            out + offset_img);
-      else if (interpolateMode == 1)
-        NNInterpolate(img, width, height, channels, x, y, out + offset_img);
-      else
-        throw "Not Implemented Interpolation Method";
-    }
-  }
-}
-
-void BilinearInterpolate(const double* img, int width, int height, int channels,
-                         double x, double y, double* out) {
-  int xx, yy, m, n, u, v, offset, offset_img, l;
-  xx = x;
-  yy = y;
-
-  double dx, dy, s;
-
-  dx = __max__(__min__(x - xx, double(1)), double(0));
-  dy = __max__(__min__(y - yy, double(1)), double(0));
-
-  for (m = 0; m <= 1; m++)
-    for (n = 0; n <= 1; n++) {
-      u = EnforceRange(yy + n, width);
-      v = EnforceRange(xx + m, height);
-      offset = v * width + u;
-      offset_img = offset * channels;
-      s = fabs(1 - m - dx) * fabs(1 - n - dy);
-      for (l = 0; l < channels; l++) out[l] += img[offset_img + l] * s;
-    }
-}
-
-void NNInterpolate(const double* img, int width, int height, int channels,
-                   double x, double y, double* out) {
-  int xx, yy, m, n, u, v, offset, offset_img, l;
-  xx = x;
-  yy = y;
-
-  double dx, dy;
-
-  dx = __max__(__min__(x - xx, double(1)), double(0));
-  dy = __max__(__min__(y - yy, double(1)), double(0));
-
-  m = (dx < 0.5) ? 0 : 1;
-  n = (dy < 0.5) ? 0 : 1;
-
-  u = EnforceRange(yy + n, width);
-  v = EnforceRange(xx + m, height);
-  offset = v * width + u;
-  offset_img = offset * channels;
-
-  for (l = 0; l < channels; l++) out[l] = img[offset_img + l];
-}

mmcv/video/optflow_warp/flow_warp.hpp

-// Copyright (c) Open-MMLab. All rights reserved.
-#include <math.h>
-#include <string.h>
-
-using namespace std;
-
-void FlowWarp(double* img, double* flow1, double* out, const int height,
-              const int width, const int channels, const int filling_value,
-              const int interpolateMode);
-
-void BilinearInterpolate(const double* img, int width, int height, int channels,
-                         double x, double y, double* out);
-
-void NNInterpolate(const double* img, int width, int height, int channels,
-                   double x, double y, double* out);
-
-template <typename T>
-inline T __min__(T a, T b) {
-  return a > b ? b : a;
-}
-
-template <typename T>
-inline T __max__(T a, T b) {
-  return (a < b) ? b : a;
-}
-
-template <typename T>
-inline T EnforceRange(const T x, const int MaxValue) {
-  return __min__(__max__(x, 0), MaxValue);
-}

mmcv/video/optflow_warp/flow_warp_module.pyx

-# Copyright (c) Open-MMLab. All rights reserved.
-# cython: language_level=3
-STUFF = "Hi"
-
-import numpy as np
-cimport numpy as np
-
-np.import_array()
-
-cdef extern from "flow_warp.hpp":
-    void FlowWarp(double* img, double* flow1, double* out, const int height, const int width, const int channels, const int filling_value, const int interpolateMode)
-
-def flow_warp_c(np.ndarray[double, ndim=3, mode="c"] img_array not None,
-                np.ndarray[double, ndim=3, mode="c"] flow_array not None,
-                int filling_value=0,
-                int interpolate_mode=1):
-
-    out_array = np.zeros_like(img_array)
-
-    FlowWarp(<double*> np.PyArray_DATA(img_array),
-             <double*> np.PyArray_DATA(flow_array),
-             <double*> np.PyArray_DATA(out_array),
-             out_array.shape[0],
-             out_array.shape[1],
-             out_array.shape[2],
-             filling_value,
-             interpolate_mode)
-
-    return out_array

setup.cfg

@@ -14,6 +14,6 @@ line_length = 79
 multi_line_output = 0
 known_standard_library = pkg_resources,setuptools
 known_first_party = mmcv
-known_third_party = Cython,addict,cv2,m2r,numpy,onnx,onnxruntime,pytest,recommonmark,resnet_cifar,torch,torchvision,yaml,yapf
+known_third_party = addict,cv2,m2r,numpy,onnx,onnxruntime,pytest,recommonmark,resnet_cifar,torch,torchvision,yaml,yapf
 no_lines_before = STDLIB,LOCALFOLDER
 default_section = THIRDPARTY

setup.py

 import glob
 import os
 import re
-import setuptools
 from pkg_resources import DistributionNotFound, get_distribution
-from setuptools import dist, find_packages, setup
-
-dist.Distribution().fetch_build_eggs(['Cython', 'numpy>=1.11.1'])
-
-import numpy  # NOQA: E402  # isort:skip
-from Cython.Build import cythonize  # NOQA: E402  # isort:skip
+from setuptools import find_packages, setup
 
 EXT_TYPE = ''
 try:
@@ -19,8 +13,9 @@ try:
     else:
         from torch.utils.cpp_extension import BuildExtension
         EXT_TYPE = 'pytorch'
+    cmd_class = {'build_ext': BuildExtension}
 except ModuleNotFoundError:
-    from Cython.Distutils import build_ext as BuildExtension
+    cmd_class = {}
     print('Skip building ext ops due to the absence of torch.')
 
 
@@ -139,16 +134,6 @@ except ImportError:
 def get_extensions():
     extensions = []
 
-    ext_flow = setuptools.Extension(
-        name='mmcv._flow_warp_ext',
-        sources=[
-            './mmcv/video/optflow_warp/flow_warp.cpp',
-            './mmcv/video/optflow_warp/flow_warp_module.pyx'
-        ],
-        include_dirs=[numpy.get_include()],
-        language='c++')
-    extensions.extend(cythonize(ext_flow))
-
     if os.getenv('MMCV_WITH_OPS', '0') == '0':
         return extensions
 
@@ -224,5 +209,5 @@ setup(
     tests_require=['pytest'],
     install_requires=install_requires,
     ext_modules=get_extensions(),
-    cmdclass={'build_ext': BuildExtension},
+    cmdclass=cmd_class,
     zip_safe=False)

tests/test_video/test_optflow.py

@@ -141,42 +141,31 @@ def test_flow2rgb():
 
 def test_flow_warp():
 
-    def np_flow_warp(flow, img):
-        output = np.zeros_like(img, dtype=img.dtype)
-        height = flow.shape[0]
-        width = flow.shape[1]
+    img = np.zeros((5, 5, 3))
+    img[2, 2, 0] = 1
+    flow = np.ones((5, 5, 2))
 
-        grid = np.indices((height, width)).swapaxes(0, 1).swapaxes(1, 2)
-        dx = grid[:, :, 0] + flow[:, :, 1]
-        dy = grid[:, :, 1] + flow[:, :, 0]
-        sx = np.floor(dx).astype(int)
-        sy = np.floor(dy).astype(int)
-        valid = (sx >= 0) & (sx < height - 1) & (sy >= 0) & (sy < width - 1)
+    res_nn = mmcv.flow_warp(img, flow, interpolate_mode='nearest')
+    res_bi = mmcv.flow_warp(img, flow, interpolate_mode='bilinear')
 
-        output[valid, :] = img[dx[valid].round().astype(int),
-                               dy[valid].round().astype(int), :]
+    assert_array_almost_equal(res_nn, res_bi, decimal=5)
 
-        return output
+    img = np.zeros((5, 5, 1))
+    img[2, 2, 0] = 1
+    img[2, 3, 0] = 0.75
+    flow = np.zeros((5, 5, 2))
+    flow[2, 2, :] = [0.5, 0.7]
 
-    dim = 500
-    a = np.random.randn(dim, dim, 3) * 10 + 125
-    b = np.random.randn(dim, dim, 2) + 2 + 0.2
+    res_ = np.copy(img)
+    res_[2, 2] = 0.5 * 0.3 + 0.75 * 0.5 * 0.3
+    res_bi = mmcv.flow_warp(img, flow, interpolate_mode='bilinear')
+    assert_array_almost_equal(res_, res_bi, decimal=5)
 
-    c = mmcv.flow_warp(a, b, interpolate_mode='nearest')
+    with pytest.raises(NotImplementedError):
+        _ = mmcv.flow_warp(img, flow, interpolate_mode='xxx')
 
-    d = np_flow_warp(b, a)
-
-    simple_a = np.zeros((5, 5, 3))
-    simple_a[2, 2, 0] = 1
-    simple_b = np.ones((5, 5, 2))
-
-    simple_res_c = np.zeros((5, 5, 3))
-    simple_res_c[1, 1, 0] = 1
-
-    res_c = mmcv.flow_warp(simple_a, simple_b, interpolate_mode='bilinear')
-
-    assert_array_equal(c, d)
-    assert_array_equal(res_c, simple_res_c)
+    with pytest.raises(AssertionError):
+        _ = mmcv.flow_warp(img, flow[:, :, 0], interpolate_mode='xxx')
 
 
 def test_make_color_wheel():