"...git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "ce315081340fdf6846f16c321eb53878b6272d53"
Unverified Commit a2c8c8ef authored by Ralph Tang's avatar Ralph Tang Committed by GitHub
Browse files

Fix hanging when loading pretrained models

- Fix hanging when loading pretrained models from the cache without having internet access. This is a widespread issue on supercomputers whose internal compute nodes are firewalled.
parent 82f6abd9
...@@ -246,7 +246,7 @@ def http_get(url, temp_file, proxies=None): ...@@ -246,7 +246,7 @@ def http_get(url, temp_file, proxies=None):
progress.close() progress.close()
def get_from_cache(url, cache_dir=None, force_download=False, proxies=None): def get_from_cache(url, cache_dir=None, force_download=False, proxies=None, etag_timeout=10):
""" """
Given a URL, look for the corresponding dataset in the local cache. Given a URL, look for the corresponding dataset in the local cache.
If it's not there, download it. Then return the path to the cached file. If it's not there, download it. Then return the path to the cached file.
...@@ -266,12 +266,12 @@ def get_from_cache(url, cache_dir=None, force_download=False, proxies=None): ...@@ -266,12 +266,12 @@ def get_from_cache(url, cache_dir=None, force_download=False, proxies=None):
etag = s3_etag(url, proxies=proxies) etag = s3_etag(url, proxies=proxies)
else: else:
try: try:
response = requests.head(url, allow_redirects=True, proxies=proxies) response = requests.head(url, allow_redirects=True, proxies=proxies, timeout=etag_timeout)
if response.status_code != 200: if response.status_code != 200:
etag = None etag = None
else: else:
etag = response.headers.get("ETag") etag = response.headers.get("ETag")
except EnvironmentError: except (EnvironmentError, requests.exceptions.Timeout):
etag = None etag = None
if sys.version_info[0] == 2 and etag is not None: if sys.version_info[0] == 2 and etag is not None:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment