fileio related interfaces should keep bc (#2539)

* fileio related interfaces should keep bc

* fix format

* update unit tests of transforms

* update format

* lazily infer client until calling get function of fileio

mmcv/image/io.py

@@ -145,6 +145,7 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
            flag: str = 'color',
            channel_order: str = 'bgr',
            backend: Optional[str] = None,
+           file_client_args: Optional[dict] = None,
            *,
            backend_args: Optional[dict] = None) -> np.ndarray:
     """Read an image.
@@ -166,12 +167,18 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
             `cv2`, `pillow`, `turbojpeg`, `tifffile`, `None`.
             If backend is None, the global imread_backend specified by
             ``mmcv.use_backend()`` will be used. Default: None.
+        file_client_args (dict, optional): Arguments to instantiate a
+            FileClient. See :class:`mmengine.fileio.FileClient` for details.
+            Default: None. It will be deprecated in future. Please use
+            ``backend_args`` instead.
+            Deprecated in version 2.0.0rc4.
         backend_args (dict, optional): Instantiates the corresponding file
             backend. It may contain `backend` key to specify the file
             backend. If it contains, the file backend corresponding to this
             value will be used and initialized with the remaining values,
             otherwise the corresponding file backend will be selected
             based on the prefix of the file path. Defaults to None.
+            New in version 2.0.0rc4.
 
     Returns:
         ndarray: Loaded image array.
@@ -195,13 +202,27 @@ def imread(img_or_path: Union[np.ndarray, str, Path],
         >>> img = mmcv.imread(http_img_path, backend_args={
         ...     'backend': 'http'})
     """
+    if file_client_args is not None:
+        warnings.warn(
+            '"file_client_args" will be deprecated in future. '
+            'Please use "backend_args" instead', DeprecationWarning)
+        if backend_args is not None:
+            raise ValueError(
+                '"file_client_args" and "backend_args" cannot be set at the '
+                'same time.')
+
     if isinstance(img_or_path, Path):
         img_or_path = str(img_or_path)
 
     if isinstance(img_or_path, np.ndarray):
         return img_or_path
     elif is_str(img_or_path):
-        img_bytes = fileio.get(img_or_path, backend_args=backend_args)
+        if file_client_args is not None:
+            file_client = fileio.FileClient.infer_client(
+                file_client_args, img_or_path)
+            img_bytes = file_client.get(img_or_path)
+        else:
+            img_bytes = fileio.get(img_or_path, backend_args=backend_args)
         return imfrombytes(img_bytes, flag, channel_order, backend)
     else:
         raise TypeError('"img" must be a numpy array or a str or '
@@ -271,6 +292,7 @@ def imwrite(img: np.ndarray,
             file_path: str,
             params: Optional[list] = None,
             auto_mkdir: Optional[bool] = None,
+            file_client_args: Optional[dict] = None,
             *,
             backend_args: Optional[dict] = None) -> bool:
     """Write image to file.
@@ -285,12 +307,18 @@ def imwrite(img: np.ndarray,
         params (None or list): Same as opencv :func:`imwrite` interface.
         auto_mkdir (bool): If the parent folder of `file_path` does not exist,
             whether to create it automatically. It will be deprecated.
+        file_client_args (dict, optional): Arguments to instantiate a
+            FileClient. See :class:`mmengine.fileio.FileClient` for details.
+            Default: None. It will be deprecated in future. Please use
+            ``backend_args`` instead.
+            Deprecated in version 2.0.0rc4.
         backend_args (dict, optional): Instantiates the corresponding file
             backend. It may contain `backend` key to specify the file
             backend. If it contains, the file backend corresponding to this
             value will be used and initialized with the remaining values,
             otherwise the corresponding file backend will be selected
             based on the prefix of the file path. Defaults to None.
+            New in version 2.0.0rc4.
 
     Returns:
         bool: Successful or not.
@@ -304,6 +332,15 @@ def imwrite(img: np.ndarray,
         >>> ret = mmcv.imwrite(img, 's3://bucket/img.jpg', backend_args={
         ...     'backend': 'petrel'})
     """
+    if file_client_args is not None:
+        warnings.warn(
+            '"file_client_args" will be deprecated in future. '
+            'Please use "backend_args" instead', DeprecationWarning)
+        if backend_args is not None:
+            raise ValueError(
+                '"file_client_args" and "backend_args" cannot be set at the '
+                'same time.')
+
     assert is_filepath(file_path)
     file_path = str(file_path)
     if auto_mkdir is not None:
@@ -317,6 +354,11 @@ def imwrite(img: np.ndarray,
     # format is '.jpg'.
     flag, img_buff = cv2.imencode(img_ext, img, params)
 
-    fileio.put(img_buff.tobytes(), file_path, backend_args=backend_args)
+    if file_client_args is not None:
+        file_client = fileio.FileClient.infer_client(file_client_args,
+                                                     file_path)
+        file_client.put(img_buff.tobytes(), file_path)
+    else:
+        fileio.put(img_buff.tobytes(), file_path, backend_args=backend_args)
 
     return flag

mmcv/transforms/loading.py

 # Copyright (c) OpenMMLab. All rights reserved.
+import warnings
 from typing import Optional
 
 import mmengine.fileio as fileio
@@ -33,6 +34,11 @@ class LoadImageFromFile(BaseTransform):
             argument for :func:`mmcv.imfrombytes`.
             See :func:`mmcv.imfrombytes` for details.
             Defaults to 'cv2'.
+        file_client_args (dict, optional): Arguments to instantiate a
+            FileClient. See :class:`mmengine.fileio.FileClient` for details.
+            Defaults to None. It will be deprecated in future. Please use
+            ``backend_args`` instead.
+            Deprecated in version 2.0.0rc4.
         ignore_empty (bool): Whether to allow loading empty image or file path
             not existent. Defaults to False.
         backend_args (dict, optional): Instantiates the corresponding file
@@ -41,12 +47,14 @@ class LoadImageFromFile(BaseTransform):
             value will be used and initialized with the remaining values,
             otherwise the corresponding file backend will be selected
             based on the prefix of the file path. Defaults to None.
+            New in version 2.0.0rc4.
     """
 
     def __init__(self,
                  to_float32: bool = False,
                  color_type: str = 'color',
                  imdecode_backend: str = 'cv2',
+                 file_client_args: Optional[dict] = None,
                  ignore_empty: bool = False,
                  *,
                  backend_args: Optional[dict] = None) -> None:
@@ -54,7 +62,21 @@ class LoadImageFromFile(BaseTransform):
         self.to_float32 = to_float32
         self.color_type = color_type
         self.imdecode_backend = imdecode_backend
-        self.backend_args = backend_args
+
+        self.file_client_args: Optional[dict] = None
+        self.backend_args: Optional[dict] = None
+        if file_client_args is not None:
+            warnings.warn(
+                '"file_client_args" will be deprecated in future. '
+                'Please use "backend_args" instead', DeprecationWarning)
+            if backend_args is not None:
+                raise ValueError(
+                    '"file_client_args" and "backend_args" cannot be set '
+                    'at the same time.')
+
+            self.file_client_args = file_client_args.copy()
+        if backend_args is not None:
+            self.backend_args = backend_args.copy()
 
     def transform(self, results: dict) -> Optional[dict]:
         """Functions to load image.
@@ -69,7 +91,13 @@ class LoadImageFromFile(BaseTransform):
 
         filename = results['img_path']
         try:
-            img_bytes = fileio.get(filename, backend_args=self.backend_args)
+            if self.file_client_args is not None:
+                file_client = fileio.FileClient.infer_client(
+                    self.file_client_args, filename)
+                img_bytes = file_client.get(filename)
+            else:
+                img_bytes = fileio.get(
+                    filename, backend_args=self.backend_args)
             img = mmcv.imfrombytes(
                 img_bytes, flag=self.color_type, backend=self.imdecode_backend)
         except Exception as e:
@@ -90,12 +118,12 @@ class LoadImageFromFile(BaseTransform):
                     f'ignore_empty={self.ignore_empty}, '
                     f'to_float32={self.to_float32}, '
                     f"color_type='{self.color_type}', "
-                    f"imdecode_backend='{self.imdecode_backend}'")
+                    f"imdecode_backend='{self.imdecode_backend}', ")
 
-        if self.backend_args is not None:
-            repr_str += f', backend_args={self.backend_args})'
+        if self.file_client_args is not None:
+            repr_str += f'file_client_args={self.file_client_args})'
         else:
-            repr_str += ')'
+            repr_str += f'backend_args={self.backend_args})'
 
         return repr_str
 
@@ -177,12 +205,18 @@ class LoadAnnotations(BaseTransform):
             argument for :func:`mmcv.imfrombytes`.
             See :func:`mmcv.imfrombytes` for details.
             Defaults to 'cv2'.
+        file_client_args (dict, optional): Arguments to instantiate a
+            FileClient. See :class:`mmengine.fileio.FileClient` for details.
+            Defaults to None. It will be deprecated in future. Please use
+            ``backend_args`` instead.
+            Deprecated in version 2.0.0rc4.
         backend_args (dict, optional): Instantiates the corresponding file
             backend. It may contain `backend` key to specify the file
             backend. If it contains, the file backend corresponding to this
             value will be used and initialized with the remaining values,
             otherwise the corresponding file backend will be selected
             based on the prefix of the file path. Defaults to None.
+            New in version 2.0.0rc4.
     """
 
     def __init__(
@@ -192,6 +226,7 @@ class LoadAnnotations(BaseTransform):
         with_seg: bool = False,
         with_keypoints: bool = False,
         imdecode_backend: str = 'cv2',
+        file_client_args: Optional[dict] = None,
         *,
         backend_args: Optional[dict] = None,
     ) -> None:
@@ -201,7 +236,21 @@ class LoadAnnotations(BaseTransform):
         self.with_seg = with_seg
         self.with_keypoints = with_keypoints
         self.imdecode_backend = imdecode_backend
-        self.backend_args = backend_args
+
+        self.file_client_args: Optional[dict] = None
+        self.backend_args: Optional[dict] = None
+        if file_client_args is not None:
+            warnings.warn(
+                '"file_client_args" will be deprecated in future. '
+                'Please use "backend_args" instead', DeprecationWarning)
+            if backend_args is not None:
+                raise ValueError(
+                    '"file_client_args" and "backend_args" cannot be set '
+                    'at the same time.')
+
+            self.file_client_args = file_client_args.copy()
+        if backend_args is not None:
+            self.backend_args = backend_args.copy()
 
     def _load_bboxes(self, results: dict) -> None:
         """Private function to load bounding box annotations.
@@ -245,9 +294,14 @@ class LoadAnnotations(BaseTransform):
         Returns:
             dict: The dict contains loaded semantic segmentation annotations.
         """
+        if self.file_client_args is not None:
+            file_client = fileio.FileClient.infer_client(
+                self.file_client_args, results['seg_map_path'])
+            img_bytes = file_client.get(results['seg_map_path'])
+        else:
+            img_bytes = fileio.get(
+                results['seg_map_path'], backend_args=self.backend_args)
 
-        img_bytes = fileio.get(
-            results['seg_map_path'], backend_args=self.backend_args)
         results['gt_seg_map'] = mmcv.imfrombytes(
             img_bytes, flag='unchanged',
             backend=self.imdecode_backend).squeeze()
@@ -296,11 +350,11 @@ class LoadAnnotations(BaseTransform):
         repr_str += f'with_label={self.with_label}, '
         repr_str += f'with_seg={self.with_seg}, '
         repr_str += f'with_keypoints={self.with_keypoints}, '
-        repr_str += f"imdecode_backend='{self.imdecode_backend}'"
+        repr_str += f"imdecode_backend='{self.imdecode_backend}', "
 
-        if self.backend_args is not None:
-            repr_str += f', backend_args={self.backend_args})'
+        if self.file_client_args is not None:
+            repr_str += f'file_client_args={self.file_client_args})'
         else:
-            repr_str += ')'
+            repr_str += f'backend_args={self.backend_args})'
 
         return repr_str

tests/test_image/test_io.py

@@ -60,6 +60,15 @@ class TestIO:
         # backend cv2
         mmcv.use_backend('cv2')
 
+        # file_client_args and backend_args can not be both set
+        with pytest.raises(
+                ValueError,
+                match='"file_client_args" and "backend_args" cannot be set'):
+            mmcv.imread(
+                self.img_path,
+                file_client_args={'backend': 'disk'},
+                backend_args={'backend': 'disk'})
+
         # HardDiskBackend
         img_cv2_color_bgr = mmcv.imread(self.img_path)
         assert img_cv2_color_bgr.shape == (300, 400, 3)
@@ -95,6 +104,16 @@ class TestIO:
                 PetrelBackend, 'get',
                 return_value=img_cv2_color_bgr) as mock_method:
             img_cv2_color_bgr_petrel = mmcv.imread(self.s3_path, backend='cv2')
+            img_cv2_color_bgr_petrel_with_args = mmcv.imread(
+                self.s3_path,
+                backend='cv2',
+                file_client_args={'backend': 'petrel'})
+            mock_method.assert_called()
+            assert_array_equal(img_cv2_color_bgr_petrel,
+                               img_cv2_color_bgr_petrel_with_args)
+
+            mock_method.reset_mock()
+
             img_cv2_color_bgr_petrel_with_args = mmcv.imread(
                 self.s3_path,
                 backend='cv2',
@@ -109,6 +128,16 @@ class TestIO:
                 HTTPBackend, 'get',
                 return_value=img_cv2_color_bgr) as mock_method:
             img_cv2_color_bgr_http = mmcv.imread(self.http_path, backend='cv2')
+            img_cv2_color_bgr_http_with_args = mmcv.imread(
+                self.http_path,
+                backend='cv2',
+                file_client_args={'backend': 'http'})
+            mock_method.assert_called()
+            assert_array_equal(img_cv2_color_bgr_http,
+                               img_cv2_color_bgr_http_with_args)
+
+            mock_method.reset_mock()
+
             img_cv2_color_bgr_http_with_args = mmcv.imread(
                 self.http_path,
                 backend='cv2',
@@ -358,6 +387,16 @@ class TestIO:
         img = mmcv.imread(self.img_path)
         out_file = osp.join(tempfile.gettempdir(), 'mmcv_test.jpg')
 
+        # file_client_args and backend_args can not be both set
+        with pytest.raises(
+                ValueError,
+                match='"file_client_args" and "backend_args" cannot be set'):
+            mmcv.imwrite(
+                img,
+                out_file,
+                file_client_args={'backend': 'disk'},
+                backend_args={'backend': 'disk'})
+
         mmcv.imwrite(img, out_file)
         rewrite_img = mmcv.imread(out_file)
         os.remove(out_file)
@@ -367,7 +406,13 @@ class TestIO:
         with patch.object(
                 PetrelBackend, 'put', return_value=None) as mock_method:
             ret = mmcv.imwrite(img, self.s3_path)
+            ret_with_args = mmcv.imwrite(
+                img, self.s3_path, file_client_args={'backend': 'petrel'})
             assert ret
+            assert ret_with_args
+            mock_method.assert_called()
+
+            mock_method.reset_mock()
 
             ret_with_args = mmcv.imwrite(
                 img, self.s3_path, backend_args={'backend': 'petrel'})

tests/test_transforms/test_transforms_loading.py

@@ -11,6 +11,13 @@ from mmcv.transforms import LoadAnnotations, LoadImageFromFile
 class TestLoadImageFromFile:
 
     def test_load_img(self):
+        # file_client_args and backend_args can not be both set
+        with pytest.raises(
+                ValueError,
+                match='"file_client_args" and "backend_args" cannot be set'):
+            LoadImageFromFile(
+                file_client_args={'backend': 'disk'},
+                backend_args={'backend': 'disk'})
         data_prefix = osp.join(osp.dirname(__file__), '../data')
 
         results = dict(img_path=osp.join(data_prefix, 'color.jpg'))
@@ -23,7 +30,7 @@ class TestLoadImageFromFile:
         assert results['ori_shape'] == (300, 400)
         assert repr(transform) == transform.__class__.__name__ + \
             "(ignore_empty=False, to_float32=False, color_type='color', " + \
-            "imdecode_backend='cv2')"
+            "imdecode_backend='cv2', backend_args=None)"
 
         # to_float32
         transform = LoadImageFromFile(to_float32=True)
@@ -71,6 +78,15 @@ class TestLoadAnnotations:
             }]
         }
 
+    def test_init(self):
+        # file_client_args and backend_args can not be both set
+        with pytest.raises(
+                ValueError,
+                match='"file_client_args" and "backend_args" cannot be set'):
+            LoadAnnotations(
+                file_client_args={'backend': 'disk'},
+                backend_args={'backend': 'disk'})
+
     def test_load_bboxes(self):
         transform = LoadAnnotations(
             with_bbox=True,
@@ -131,4 +147,5 @@ class TestLoadAnnotations:
         assert repr(transform) == (
             'LoadAnnotations(with_bbox=True, '
             'with_label=False, with_seg=False, '
-            "with_keypoints=False, imdecode_backend='cv2')")
+            "with_keypoints=False, imdecode_backend='cv2', "
+            'backend_args=None)')