Skip to content

GitLab

Unverified Commit 009cfe60 authored by Zaida Zhou's avatar Zaida Zhou Committed by GitHub
Browse files

fileio related interfaces should keep bc (#2539) 

* fileio related interfaces should keep bc

* fix format

* update unit tests of transforms

* update format

* lazily infer client until calling get function of fileio
parent 34bdf448
Show whitespace changes
Inline Side-by-side
Showing with 175 additions and 17 deletions
mmcv/image/io.py
View file @ 009cfe60
......@@ -145,6 +145,7 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
flag: str = 'color',
channel_order: str = 'bgr',
backend: Optional[str] = None,
file_client_args: Optional[dict] = None,
*,
backend_args: Optional[dict] = None) -> np.ndarray:
"""Read an image.
......@@ -166,12 +167,18 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
`cv2`, `pillow`, `turbojpeg`, `tifffile`, `None`.
If backend is None, the global imread_backend specified by
``mmcv.use_backend()`` will be used. Default: None.
file_client_args (dict, optional): Arguments to instantiate a
FileClient. See :class:`mmengine.fileio.FileClient` for details.
Default: None. It will be deprecated in future. Please use
``backend_args`` instead.
Deprecated in version 2.0.0rc4.
backend_args (dict, optional): Instantiates the corresponding file
backend. It may contain `backend` key to specify the file
backend. If it contains, the file backend corresponding to this
value will be used and initialized with the remaining values,
otherwise the corresponding file backend will be selected
based on the prefix of the file path. Defaults to None.
New in version 2.0.0rc4.
Returns:
ndarray: Loaded image array.
......@@ -195,12 +202,26 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
>>> img = mmcv.imread(http_img_path, backend_args={
... 'backend': 'http'})
"""
if file_client_args is not None:
warnings.warn(
'"file_client_args" will be deprecated in future. '
'Please use "backend_args" instead', DeprecationWarning)
if backend_args is not None:
raise ValueError(
'"file_client_args" and "backend_args" cannot be set at the '
'same time.')
if isinstance(img_or_path, Path):
img_or_path = str(img_or_path)
if isinstance(img_or_path, np.ndarray):
return img_or_path
elif is_str(img_or_path):
if file_client_args is not None:
file_client = fileio.FileClient.infer_client(
file_client_args, img_or_path)
img_bytes = file_client.get(img_or_path)
else:
img_bytes = fileio.get(img_or_path, backend_args=backend_args)
return imfrombytes(img_bytes, flag, channel_order, backend)
else:
......@@ -271,6 +292,7 @@ def imwrite(img: np.ndarray,
file_path: str,
params: Optional[list] = None,
auto_mkdir: Optional[bool] = None,
file_client_args: Optional[dict] = None,
*,
backend_args: Optional[dict] = None) -> bool:
"""Write image to file.
......@@ -285,12 +307,18 @@ def imwrite(img: np.ndarray,
params (None or list): Same as opencv :func:`imwrite` interface.
auto_mkdir (bool): If the parent folder of `file_path` does not exist,
whether to create it automatically. It will be deprecated.
file_client_args (dict, optional): Arguments to instantiate a
FileClient. See :class:`mmengine.fileio.FileClient` for details.
Default: None. It will be deprecated in future. Please use
``backend_args`` instead.
Deprecated in version 2.0.0rc4.
backend_args (dict, optional): Instantiates the corresponding file
backend. It may contain `backend` key to specify the file
backend. If it contains, the file backend corresponding to this
value will be used and initialized with the remaining values,
otherwise the corresponding file backend will be selected
based on the prefix of the file path. Defaults to None.
New in version 2.0.0rc4.
Returns:
bool: Successful or not.
......@@ -304,6 +332,15 @@ def imwrite(img: np.ndarray,
>>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg', backend_args={
... 'backend': 'petrel'})
"""
if file_client_args is not None:
warnings.warn(
'"file_client_args" will be deprecated in future. '
'Please use "backend_args" instead', DeprecationWarning)
if backend_args is not None:
raise ValueError(
'"file_client_args" and "backend_args" cannot be set at the '
'same time.')
assert is_filepath(file_path)
file_path = str(file_path)
if auto_mkdir is not None:
......@@ -317,6 +354,11 @@ def imwrite(img: np.ndarray,
# format is '.jpg'.
flag, img_buff = cv2.imencode(img_ext, img, params)
if file_client_args is not None:
file_client = fileio.FileClient.infer_client(file_client_args,
file_path)
file_client.put(img_buff.tobytes(), file_path)
else:
fileio.put(img_buff.tobytes(), file_path, backend_args=backend_args)
return flag
mmcv/transforms/loading.py
View file @ 009cfe60
# Copyright (c) OpenMMLab. All rights reserved.
import warnings
from typing import Optional
import mmengine.fileio as fileio
......@@ -33,6 +34,11 @@ class LoadImageFromFile(BaseTransform):
argument for :func:`mmcv.imfrombytes`.
See :func:`mmcv.imfrombytes` for details.
Defaults to 'cv2'.
file_client_args (dict, optional): Arguments to instantiate a
FileClient. See :class:`mmengine.fileio.FileClient` for details.
Defaults to None. It will be deprecated in future. Please use
``backend_args`` instead.
Deprecated in version 2.0.0rc4.
ignore_empty (bool): Whether to allow loading empty image or file path
not existent. Defaults to False.
backend_args (dict, optional): Instantiates the corresponding file
......@@ -41,12 +47,14 @@ class LoadImageFromFile(BaseTransform):
value will be used and initialized with the remaining values,
otherwise the corresponding file backend will be selected
based on the prefix of the file path. Defaults to None.
New in version 2.0.0rc4.
"""
def __init__(self,
to_float32: bool = False,
color_type: str = 'color',
imdecode_backend: str = 'cv2',
file_client_args: Optional[dict] = None,
ignore_empty: bool = False,
*,
backend_args: Optional[dict] = None) -> None:
......@@ -54,7 +62,21 @@ class LoadImageFromFile(BaseTransform):
self.to_float32 = to_float32
self.color_type = color_type
self.imdecode_backend = imdecode_backend
self.backend_args = backend_args
self.file_client_args: Optional[dict] = None
self.backend_args: Optional[dict] = None
if file_client_args is not None:
warnings.warn(
'"file_client_args" will be deprecated in future. '
'Please use "backend_args" instead', DeprecationWarning)
if backend_args is not None:
raise ValueError(
'"file_client_args" and "backend_args" cannot be set '
'at the same time.')
self.file_client_args = file_client_args.copy()
if backend_args is not None:
self.backend_args = backend_args.copy()
def transform(self, results: dict) -> Optional[dict]:
"""Functions to load image.
......@@ -69,7 +91,13 @@ class LoadImageFromFile(BaseTransform):
filename = results['img_path']
try:
img_bytes = fileio.get(filename, backend_args=self.backend_args)
if self.file_client_args is not None:
file_client = fileio.FileClient.infer_client(
self.file_client_args, filename)
img_bytes = file_client.get(filename)
else:
img_bytes = fileio.get(
filename, backend_args=self.backend_args)
img = mmcv.imfrombytes(
img_bytes, flag=self.color_type, backend=self.imdecode_backend)
except Exception as e:
......@@ -90,12 +118,12 @@ class LoadImageFromFile(BaseTransform):
f'ignore_empty={self.ignore_empty}, '
f'to_float32={self.to_float32}, '
f"color_type='{self.color_type}', "
f"imdecode_backend='{self.imdecode_backend}'")
f"imdecode_backend='{self.imdecode_backend}', ")
if self.backend_args is not None:
repr_str += f', backend_args={self.backend_args})'
if self.file_client_args is not None:
repr_str += f'file_client_args={self.file_client_args})'
else:
repr_str += ')'
repr_str += f'backend_args={self.backend_args})'
return repr_str
......@@ -177,12 +205,18 @@ class LoadAnnotations(BaseTransform):
argument for :func:`mmcv.imfrombytes`.
See :func:`mmcv.imfrombytes` for details.
Defaults to 'cv2'.
file_client_args (dict, optional): Arguments to instantiate a
FileClient. See :class:`mmengine.fileio.FileClient` for details.
Defaults to None. It will be deprecated in future. Please use
``backend_args`` instead.
Deprecated in version 2.0.0rc4.
backend_args (dict, optional): Instantiates the corresponding file
backend. It may contain `backend` key to specify the file
backend. If it contains, the file backend corresponding to this
value will be used and initialized with the remaining values,
otherwise the corresponding file backend will be selected
based on the prefix of the file path. Defaults to None.
New in version 2.0.0rc4.
"""
def __init__(
......@@ -192,6 +226,7 @@ class LoadAnnotations(BaseTransform):
with_seg: bool = False,
with_keypoints: bool = False,
imdecode_backend: str = 'cv2',
file_client_args: Optional[dict] = None,
*,
backend_args: Optional[dict] = None,
) -> None:
......@@ -201,7 +236,21 @@ class LoadAnnotations(BaseTransform):
self.with_seg = with_seg
self.with_keypoints = with_keypoints
self.imdecode_backend = imdecode_backend
self.backend_args = backend_args
self.file_client_args: Optional[dict] = None
self.backend_args: Optional[dict] = None
if file_client_args is not None:
warnings.warn(
'"file_client_args" will be deprecated in future. '
'Please use "backend_args" instead', DeprecationWarning)
if backend_args is not None:
raise ValueError(
'"file_client_args" and "backend_args" cannot be set '
'at the same time.')
self.file_client_args = file_client_args.copy()
if backend_args is not None:
self.backend_args = backend_args.copy()
def _load_bboxes(self, results: dict) -> None:
"""Private function to load bounding box annotations.
......@@ -245,9 +294,14 @@ class LoadAnnotations(BaseTransform):
Returns:
dict: The dict contains loaded semantic segmentation annotations.
"""
if self.file_client_args is not None:
file_client = fileio.FileClient.infer_client(
self.file_client_args, results['seg_map_path'])
img_bytes = file_client.get(results['seg_map_path'])
else:
img_bytes = fileio.get(
results['seg_map_path'], backend_args=self.backend_args)
results['gt_seg_map'] = mmcv.imfrombytes(
img_bytes, flag='unchanged',
backend=self.imdecode_backend).squeeze()
......@@ -296,11 +350,11 @@ class LoadAnnotations(BaseTransform):
repr_str += f'with_label={self.with_label}, '
repr_str += f'with_seg={self.with_seg}, '
repr_str += f'with_keypoints={self.with_keypoints}, '
repr_str += f"imdecode_backend='{self.imdecode_backend}'"
repr_str += f"imdecode_backend='{self.imdecode_backend}', "
if self.backend_args is not None:
repr_str += f', backend_args={self.backend_args})'
if self.file_client_args is not None:
repr_str += f'file_client_args={self.file_client_args})'
else:
repr_str += ')'
repr_str += f'backend_args={self.backend_args})'
return repr_str
tests/test_image/test_io.py
View file @ 009cfe60
......@@ -60,6 +60,15 @@ class TestIO:
# backend cv2
mmcv.use_backend('cv2')
# file_client_args and backend_args can not be both set
with pytest.raises(
ValueError,
match='"file_client_args" and "backend_args" cannot be set'):
mmcv.imread(
self.img_path,
file_client_args={'backend': 'disk'},
backend_args={'backend': 'disk'})
# HardDiskBackend
img_cv2_color_bgr = mmcv.imread(self.img_path)
assert img_cv2_color_bgr.shape == (300, 400, 3)
......@@ -95,6 +104,16 @@ class TestIO:
PetrelBackend, 'get',
return_value=img_cv2_color_bgr) as mock_method:
img_cv2_color_bgr_petrel = mmcv.imread(self.s3_path, backend='cv2')
img_cv2_color_bgr_petrel_with_args = mmcv.imread(
self.s3_path,
backend='cv2',
file_client_args={'backend': 'petrel'})
mock_method.assert_called()
assert_array_equal(img_cv2_color_bgr_petrel,
img_cv2_color_bgr_petrel_with_args)
mock_method.reset_mock()
img_cv2_color_bgr_petrel_with_args = mmcv.imread(
self.s3_path,
backend='cv2',
......@@ -109,6 +128,16 @@ class TestIO:
HTTPBackend, 'get',
return_value=img_cv2_color_bgr) as mock_method:
img_cv2_color_bgr_http = mmcv.imread(self.http_path, backend='cv2')
img_cv2_color_bgr_http_with_args = mmcv.imread(
self.http_path,
backend='cv2',
file_client_args={'backend': 'http'})
mock_method.assert_called()
assert_array_equal(img_cv2_color_bgr_http,
img_cv2_color_bgr_http_with_args)
mock_method.reset_mock()
img_cv2_color_bgr_http_with_args = mmcv.imread(
self.http_path,
backend='cv2',
......@@ -358,6 +387,16 @@ class TestIO:
img = mmcv.imread(self.img_path)
out_file = osp.join(tempfile.gettempdir(), 'mmcv_test.jpg')
# file_client_args and backend_args can not be both set
with pytest.raises(
ValueError,
match='"file_client_args" and "backend_args" cannot be set'):
mmcv.imwrite(
img,
out_file,
file_client_args={'backend': 'disk'},
backend_args={'backend': 'disk'})
mmcv.imwrite(img, out_file)
rewrite_img = mmcv.imread(out_file)
os.remove(out_file)
......@@ -367,7 +406,13 @@ class TestIO:
with patch.object(
PetrelBackend, 'put', return_value=None) as mock_method:
ret = mmcv.imwrite(img, self.s3_path)
ret_with_args = mmcv.imwrite(
img, self.s3_path, file_client_args={'backend': 'petrel'})
assert ret
assert ret_with_args
mock_method.assert_called()
mock_method.reset_mock()
ret_with_args = mmcv.imwrite(
img, self.s3_path, backend_args={'backend': 'petrel'})
......
tests/test_transforms/test_transforms_loading.py
View file @ 009cfe60
......@@ -11,6 +11,13 @@ from mmcv.transforms import LoadAnnotations, LoadImageFromFile
class TestLoadImageFromFile:
def test_load_img(self):
# file_client_args and backend_args can not be both set
with pytest.raises(
ValueError,
match='"file_client_args" and "backend_args" cannot be set'):
LoadImageFromFile(
file_client_args={'backend': 'disk'},
backend_args={'backend': 'disk'})
data_prefix = osp.join(osp.dirname(__file__), '../data')
results = dict(img_path=osp.join(data_prefix, 'color.jpg'))
......@@ -23,7 +30,7 @@ class TestLoadImageFromFile:
assert results['ori_shape'] == (300, 400)
assert repr(transform) == transform.__class__.__name__ + \
"(ignore_empty=False, to_float32=False, color_type='color', " + \
"imdecode_backend='cv2')"
"imdecode_backend='cv2', backend_args=None)"
# to_float32
transform = LoadImageFromFile(to_float32=True)
......@@ -71,6 +78,15 @@ class TestLoadAnnotations:
}]
}
def test_init(self):
# file_client_args and backend_args can not be both set
with pytest.raises(
ValueError,
match='"file_client_args" and "backend_args" cannot be set'):
LoadAnnotations(
file_client_args={'backend': 'disk'},
backend_args={'backend': 'disk'})
def test_load_bboxes(self):
transform = LoadAnnotations(
with_bbox=True,
......@@ -131,4 +147,5 @@ class TestLoadAnnotations:
assert repr(transform) == (
'LoadAnnotations(with_bbox=True, '
'with_label=False, with_seg=False, '
"with_keypoints=False, imdecode_backend='cv2')")
"with_keypoints=False, imdecode_backend='cv2', "
'backend_args=None)')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment