You need to sign in or sign up before continuing.
Unverified Commit 84c7dc34 authored by Zaida Zhou's avatar Zaida Zhou Committed by GitHub
Browse files

[Fix] Fix the path of ceph containing cluster name (#1577)

* [Fix] Fix the path of ceph containing cluster name

* fix corner cases

* fix format

* add note
parent 22e73d69
...@@ -220,11 +220,13 @@ class CheckpointLoader: ...@@ -220,11 +220,13 @@ class CheckpointLoader:
path (str): checkpoint path path (str): checkpoint path
Returns: Returns:
loader (function): checkpoint loader callable: checkpoint loader
""" """
for p in cls._schemes: for p in cls._schemes:
if path.startswith(p): # use regular match to handle some cases that where the prefix of
# loader has a prefix. For example, both 's3://path' and
# 'open-mmlab:s3://path' should return `load_from_ceph`
if re.match(p, path) is not None:
return cls._schemes[p] return cls._schemes[p]
@classmethod @classmethod
...@@ -326,12 +328,17 @@ def load_from_pavi(filename, map_location=None): ...@@ -326,12 +328,17 @@ def load_from_pavi(filename, map_location=None):
return checkpoint return checkpoint
@CheckpointLoader.register_scheme(prefixes='s3://') @CheckpointLoader.register_scheme(prefixes=r'(\S+\:)?s3://')
def load_from_ceph(filename, map_location=None, backend='petrel'): def load_from_ceph(filename, map_location=None, backend='petrel'):
"""load checkpoint through the file path prefixed with s3. In distributed """load checkpoint through the file path prefixed with s3. In distributed
setting, this function download ckpt at all ranks to different temporary setting, this function download ckpt at all ranks to different temporary
directories. directories.
Note:
Since v1.4.1, the registered scheme prefixes have been enhanced to
support bucket names in the path prefix, e.g. 's3://xx.xx/xx.path',
'bucket1:s3://xx.xx/xx.path'.
Args: Args:
filename (str): checkpoint file path with s3 prefix filename (str): checkpoint file path with s3 prefix
map_location (str, optional): Same as :func:`torch.load`. map_location (str, optional): Same as :func:`torch.load`.
......
...@@ -347,13 +347,16 @@ def test_checkpoint_loader(): ...@@ -347,13 +347,16 @@ def test_checkpoint_loader():
'modelzoo://xx.xx/xx.pth', 'torchvision://xx.xx/xx.pth', 'modelzoo://xx.xx/xx.pth', 'torchvision://xx.xx/xx.pth',
'open-mmlab://xx.xx/xx.pth', 'openmmlab://xx.xx/xx.pth', 'open-mmlab://xx.xx/xx.pth', 'openmmlab://xx.xx/xx.pth',
'mmcls://xx.xx/xx.pth', 'pavi://xx.xx/xx.pth', 's3://xx.xx/xx.pth', 'mmcls://xx.xx/xx.pth', 'pavi://xx.xx/xx.pth', 's3://xx.xx/xx.pth',
'ss3://xx.xx/xx.pth', ' s3://xx.xx/xx.pth' 'ss3://xx.xx/xx.pth', ' s3://xx.xx/xx.pth',
'open-mmlab:s3://xx.xx/xx.pth', 'openmmlab:s3://xx.xx/xx.pth',
'openmmlabs3://xx.xx/xx.pth', ':s3://xx.xx/xx.path'
] ]
fn_names = [ fn_names = [
'load_from_http', 'load_from_http', 'load_from_torchvision', 'load_from_http', 'load_from_http', 'load_from_torchvision',
'load_from_torchvision', 'load_from_openmmlab', 'load_from_openmmlab', 'load_from_torchvision', 'load_from_openmmlab', 'load_from_openmmlab',
'load_from_mmcls', 'load_from_pavi', 'load_from_ceph', 'load_from_mmcls', 'load_from_pavi', 'load_from_ceph',
'load_from_local', 'load_from_local' 'load_from_local', 'load_from_local', 'load_from_ceph',
'load_from_ceph', 'load_from_local', 'load_from_local'
] ]
for filename, fn_name in zip(filenames, fn_names): for filename, fn_name in zip(filenames, fn_names):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment