Unverified Commit 84c7dc34 authored by Zaida Zhou's avatar Zaida Zhou Committed by GitHub
Browse files

[Fix] Fix the path of ceph containing cluster name (#1577)

* [Fix] Fix the path of ceph containing cluster name

* fix corner cases

* fix format

* add note
parent 22e73d69
......@@ -220,11 +220,13 @@ class CheckpointLoader:
path (str): checkpoint path
Returns:
loader (function): checkpoint loader
callable: checkpoint loader
"""
for p in cls._schemes:
if path.startswith(p):
# use regular match to handle some cases that where the prefix of
# loader has a prefix. For example, both 's3://path' and
# 'open-mmlab:s3://path' should return `load_from_ceph`
if re.match(p, path) is not None:
return cls._schemes[p]
@classmethod
......@@ -326,12 +328,17 @@ def load_from_pavi(filename, map_location=None):
return checkpoint
@CheckpointLoader.register_scheme(prefixes='s3://')
@CheckpointLoader.register_scheme(prefixes=r'(\S+\:)?s3://')
def load_from_ceph(filename, map_location=None, backend='petrel'):
"""load checkpoint through the file path prefixed with s3. In distributed
setting, this function download ckpt at all ranks to different temporary
directories.
Note:
Since v1.4.1, the registered scheme prefixes have been enhanced to
support bucket names in the path prefix, e.g. 's3://xx.xx/xx.path',
'bucket1:s3://xx.xx/xx.path'.
Args:
filename (str): checkpoint file path with s3 prefix
map_location (str, optional): Same as :func:`torch.load`.
......
......@@ -347,13 +347,16 @@ def test_checkpoint_loader():
'modelzoo://xx.xx/xx.pth', 'torchvision://xx.xx/xx.pth',
'open-mmlab://xx.xx/xx.pth', 'openmmlab://xx.xx/xx.pth',
'mmcls://xx.xx/xx.pth', 'pavi://xx.xx/xx.pth', 's3://xx.xx/xx.pth',
'ss3://xx.xx/xx.pth', ' s3://xx.xx/xx.pth'
'ss3://xx.xx/xx.pth', ' s3://xx.xx/xx.pth',
'open-mmlab:s3://xx.xx/xx.pth', 'openmmlab:s3://xx.xx/xx.pth',
'openmmlabs3://xx.xx/xx.pth', ':s3://xx.xx/xx.path'
]
fn_names = [
'load_from_http', 'load_from_http', 'load_from_torchvision',
'load_from_torchvision', 'load_from_openmmlab', 'load_from_openmmlab',
'load_from_mmcls', 'load_from_pavi', 'load_from_ceph',
'load_from_local', 'load_from_local'
'load_from_local', 'load_from_local', 'load_from_ceph',
'load_from_ceph', 'load_from_local', 'load_from_local'
]
for filename, fn_name in zip(filenames, fn_names):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment