Unverified Commit cb781ae4 authored by Wencheng Wu's avatar Wencheng Wu Committed by GitHub
Browse files

[Fix] Fix the problem that the instance cannot reuse when overwriting the file client. (#1747)

* [Fix] Fix the problem that the instance cannot reuse when overwriting the file client.

* Remove _overridden_backends and _overridden_prefixes, and pop the old instance when overwriting the file client.

* add unittest

* fix prefix error

* modify clear_backend to overridden_backend

* Delete redundant comments.
parent 544a6b14
...@@ -791,17 +791,12 @@ class FileClient: ...@@ -791,17 +791,12 @@ class FileClient:
'petrel': PetrelBackend, 'petrel': PetrelBackend,
'http': HTTPBackend, 'http': HTTPBackend,
} }
# This collection is used to record the overridden backends, and when a
# backend appears in the collection, the singleton pattern is disabled for _prefix_to_backends = {
# that backend, because if the singleton pattern is used, then the object
# returned will be the backend before overwriting
_overridden_backends: set = set()
_prefix_to_backends: dict = {
's3': PetrelBackend, 's3': PetrelBackend,
'http': HTTPBackend, 'http': HTTPBackend,
'https': HTTPBackend, 'https': HTTPBackend,
} }
_overridden_prefixes: set = set()
_instances: dict = {} _instances: dict = {}
...@@ -825,10 +820,7 @@ class FileClient: ...@@ -825,10 +820,7 @@ class FileClient:
for key, value in kwargs.items(): for key, value in kwargs.items():
arg_key += f':{key}:{value}' arg_key += f':{key}:{value}'
# if a backend was overridden, it will create a new object if arg_key in cls._instances:
if (arg_key in cls._instances
and backend not in cls._overridden_backends
and prefix not in cls._overridden_prefixes):
_instance = cls._instances[arg_key] _instance = cls._instances[arg_key]
else: else:
# create a new object and put it to _instance # create a new object and put it to _instance
...@@ -922,7 +914,9 @@ class FileClient: ...@@ -922,7 +914,9 @@ class FileClient:
'add "force=True" if you want to override it') 'add "force=True" if you want to override it')
if name in cls._backends and force: if name in cls._backends and force:
cls._overridden_backends.add(name) for arg_key, instance in list(cls._instances.items()):
if isinstance(instance.client, cls._backends[name]):
cls._instances.pop(arg_key)
cls._backends[name] = backend cls._backends[name] = backend
if prefixes is not None: if prefixes is not None:
...@@ -934,7 +928,12 @@ class FileClient: ...@@ -934,7 +928,12 @@ class FileClient:
if prefix not in cls._prefix_to_backends: if prefix not in cls._prefix_to_backends:
cls._prefix_to_backends[prefix] = backend cls._prefix_to_backends[prefix] = backend
elif (prefix in cls._prefix_to_backends) and force: elif (prefix in cls._prefix_to_backends) and force:
cls._overridden_prefixes.add(prefix) overridden_backend = cls._prefix_to_backends[prefix]
if isinstance(overridden_backend, list):
overridden_backend = tuple(overridden_backend)
for arg_key, instance in list(cls._instances.items()):
if isinstance(instance.client, overridden_backend):
cls._instances.pop(arg_key)
cls._prefix_to_backends[prefix] = backend cls._prefix_to_backends[prefix] = backend
else: else:
raise KeyError( raise KeyError(
......
...@@ -655,7 +655,8 @@ class TestFileClient: ...@@ -655,7 +655,8 @@ class TestFileClient:
FileClient.register_backend('dummy_backend', DummyBackend2, force=True) FileClient.register_backend('dummy_backend', DummyBackend2, force=True)
client3 = FileClient(backend='dummy_backend') client3 = FileClient(backend='dummy_backend')
client4 = FileClient(backend='dummy_backend') client4 = FileClient(backend='dummy_backend')
assert client3 is not client4 assert client2 is not client3
assert client3 is client4
def test_parse_uri_prefix(self): def test_parse_uri_prefix(self):
# input path is None # input path is None
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment