Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
OpenDAS
MMCV
Commits
3bf3e8ef
Unverified
Commit
3bf3e8ef
authored
May 23, 2020
by
Kai Chen
Committed by
GitHub
May 23, 2020
Browse files
Revert "add PetrelBackend (#291)" (#293)
This reverts commit
7f80a2c6
.
parent
7f80a2c6
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
2 additions
and
103 deletions
+2
-103
mmcv/fileio/file_client.py
mmcv/fileio/file_client.py
+2
-49
tests/test_fileclient.py
tests/test_fileclient.py
+0
-54
No files found.
mmcv/fileio/file_client.py
View file @
3bf3e8ef
import
inspect
import
inspect
import
warnings
from
abc
import
ABCMeta
,
abstractmethod
from
abc
import
ABCMeta
,
abstractmethod
...
@@ -21,63 +20,18 @@ class BaseStorageBackend(metaclass=ABCMeta):
...
@@ -21,63 +20,18 @@ class BaseStorageBackend(metaclass=ABCMeta):
class
CephBackend
(
BaseStorageBackend
):
class
CephBackend
(
BaseStorageBackend
):
"""Ceph storage backend.
"""Ceph storage backend.
"""
Args:
def
__init__
(
self
):
path_mapping (dict|None): path mapping dict from local path to Petrel
path. When `path_mapping={'src': 'dst'}`, `src` in `filepath` will
be replaced by `dst`. Default: None.
"""
def
__init__
(
self
,
path_mapping
=
None
):
try
:
try
:
import
ceph
import
ceph
warnings
.
warn
(
'Ceph is deprecate in favor of Petrel.'
)
except
ImportError
:
except
ImportError
:
raise
ImportError
(
'Please install ceph to enable CephBackend.'
)
raise
ImportError
(
'Please install ceph to enable CephBackend.'
)
self
.
_client
=
ceph
.
S3Client
()
self
.
_client
=
ceph
.
S3Client
()
assert
isinstance
(
path_mapping
,
dict
)
or
path_mapping
is
None
self
.
path_mapping
=
path_mapping
def
get
(
self
,
filepath
):
filepath
=
str
(
filepath
)
if
self
.
path_mapping
is
not
None
:
for
k
,
v
in
self
.
path_mapping
.
items
():
filepath
=
filepath
.
replace
(
k
,
v
)
value
=
self
.
_client
.
Get
(
filepath
)
value_buf
=
memoryview
(
value
)
return
value_buf
def
get_text
(
self
,
filepath
):
raise
NotImplementedError
class
PetrelBackend
(
BaseStorageBackend
):
"""Petrel storage backend (for internal use).
Args:
path_mapping (dict|None): path mapping dict from local path to Petrel
path. When `path_mapping={'src': 'dst'}`, `src` in `filepath` will
be replaced by `dst`. Default: None.
"""
def
__init__
(
self
,
path_mapping
=
None
):
try
:
import
petrel_client
except
ImportError
:
raise
ImportError
(
'Please install petrel_client to enable '
'PetrelBackend.'
)
self
.
_client
=
petrel_client
.
client
.
Client
()
assert
isinstance
(
path_mapping
,
dict
)
or
path_mapping
is
None
self
.
path_mapping
=
path_mapping
def
get
(
self
,
filepath
):
def
get
(
self
,
filepath
):
filepath
=
str
(
filepath
)
filepath
=
str
(
filepath
)
if
self
.
path_mapping
is
not
None
:
for
k
,
v
in
self
.
path_mapping
.
items
():
filepath
=
filepath
.
replace
(
k
,
v
)
value
=
self
.
_client
.
Get
(
filepath
)
value
=
self
.
_client
.
Get
(
filepath
)
value_buf
=
memoryview
(
value
)
value_buf
=
memoryview
(
value
)
return
value_buf
return
value_buf
...
@@ -210,7 +164,6 @@ class FileClient(object):
...
@@ -210,7 +164,6 @@ class FileClient(object):
'ceph'
:
CephBackend
,
'ceph'
:
CephBackend
,
'memcached'
:
MemcachedBackend
,
'memcached'
:
MemcachedBackend
,
'lmdb'
:
LmdbBackend
,
'lmdb'
:
LmdbBackend
,
'petrel'
:
PetrelBackend
,
}
}
def
__init__
(
self
,
backend
=
'disk'
,
**
kwargs
):
def
__init__
(
self
,
backend
=
'disk'
,
**
kwargs
):
...
...
tests/test_fileclient.py
View file @
3bf3e8ef
...
@@ -8,7 +8,6 @@ import mmcv
...
@@ -8,7 +8,6 @@ import mmcv
from
mmcv
import
BaseStorageBackend
,
FileClient
from
mmcv
import
BaseStorageBackend
,
FileClient
sys
.
modules
[
'ceph'
]
=
MagicMock
()
sys
.
modules
[
'ceph'
]
=
MagicMock
()
sys
.
modules
[
'petrel_client'
]
=
MagicMock
()
sys
.
modules
[
'mc'
]
=
MagicMock
()
sys
.
modules
[
'mc'
]
=
MagicMock
()
...
@@ -62,9 +61,6 @@ class TestFileClient(object):
...
@@ -62,9 +61,6 @@ class TestFileClient(object):
@
patch
(
'ceph.S3Client'
,
MockS3Client
)
@
patch
(
'ceph.S3Client'
,
MockS3Client
)
def
test_ceph_backend
(
self
):
def
test_ceph_backend
(
self
):
with
pytest
.
warns
(
Warning
,
match
=
'Ceph is deprecate in favor of Petrel.'
):
FileClient
(
'ceph'
)
ceph_backend
=
FileClient
(
'ceph'
)
ceph_backend
=
FileClient
(
'ceph'
)
# input path is Path object
# input path is Path object
...
@@ -83,56 +79,6 @@ class TestFileClient(object):
...
@@ -83,56 +79,6 @@ class TestFileClient(object):
img
=
mmcv
.
imfrombytes
(
img_bytes
)
img
=
mmcv
.
imfrombytes
(
img_bytes
)
assert
img
.
shape
==
self
.
img_shape
assert
img
.
shape
==
self
.
img_shape
# `path_mapping` is either None or dict
with
pytest
.
raises
(
AssertionError
):
FileClient
(
'ceph'
,
path_mapping
=
1
)
# test `path_mapping`
ceph_path
=
's3://user/data'
ceph_backend
=
FileClient
(
'ceph'
,
path_mapping
=
{
str
(
self
.
test_data_dir
):
ceph_path
})
ceph_backend
.
client
.
_client
.
Get
=
MagicMock
(
return_value
=
ceph_backend
.
client
.
_client
.
Get
(
self
.
img_path
))
img_bytes
=
ceph_backend
.
get
(
self
.
img_path
)
img
=
mmcv
.
imfrombytes
(
img_bytes
)
assert
img
.
shape
==
self
.
img_shape
ceph_backend
.
client
.
_client
.
Get
.
assert_called_with
(
str
(
self
.
img_path
).
replace
(
str
(
self
.
test_data_dir
),
ceph_path
))
@
patch
(
'petrel_client.client.Client'
,
MockS3Client
)
def
test_petrel_backend
(
self
):
petrel_backend
=
FileClient
(
'petrel'
)
# input path is Path object
with
pytest
.
raises
(
NotImplementedError
):
petrel_backend
.
get_text
(
self
.
text_path
)
# input path is str
with
pytest
.
raises
(
NotImplementedError
):
petrel_backend
.
get_text
(
str
(
self
.
text_path
))
# input path is Path object
img_bytes
=
petrel_backend
.
get
(
self
.
img_path
)
img
=
mmcv
.
imfrombytes
(
img_bytes
)
assert
img
.
shape
==
self
.
img_shape
# input path is str
img_bytes
=
petrel_backend
.
get
(
str
(
self
.
img_path
))
img
=
mmcv
.
imfrombytes
(
img_bytes
)
assert
img
.
shape
==
self
.
img_shape
# `path_mapping` is either None or dict
with
pytest
.
raises
(
AssertionError
):
FileClient
(
'petrel'
,
path_mapping
=
1
)
# test `path_mapping`
petrel_path
=
's3://user/data'
petrel_backend
=
FileClient
(
'petrel'
,
path_mapping
=
{
str
(
self
.
test_data_dir
):
petrel_path
})
petrel_backend
.
client
.
_client
.
Get
=
MagicMock
(
return_value
=
petrel_backend
.
client
.
_client
.
Get
(
self
.
img_path
))
img_bytes
=
petrel_backend
.
get
(
self
.
img_path
)
img
=
mmcv
.
imfrombytes
(
img_bytes
)
assert
img
.
shape
==
self
.
img_shape
petrel_backend
.
client
.
_client
.
Get
.
assert_called_with
(
str
(
self
.
img_path
).
replace
(
str
(
self
.
test_data_dir
),
petrel_path
))
@
patch
(
'mc.MemcachedClient.GetInstance'
,
MockMemcachedClient
)
@
patch
(
'mc.MemcachedClient.GetInstance'
,
MockMemcachedClient
)
@
patch
(
'mc.pyvector'
,
MagicMock
)
@
patch
(
'mc.pyvector'
,
MagicMock
)
@
patch
(
'mc.ConvertBuffer'
,
lambda
x
:
x
.
content
)
@
patch
(
'mc.ConvertBuffer'
,
lambda
x
:
x
.
content
)
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment