Unverified Commit c645f9d2 authored by Philip Meier's avatar Philip Meier Committed by GitHub
Browse files

Improve download tests (#3346)



* make timeout a parameter

* check redirects
Co-authored-by: default avatarFrancisco Massa <fvsmassa@gmail.com>
parent 82ae5d1f
...@@ -7,6 +7,7 @@ from os import path ...@@ -7,6 +7,7 @@ from os import path
from urllib.error import HTTPError, URLError from urllib.error import HTTPError, URLError
from urllib.parse import urlparse from urllib.parse import urlparse
from urllib.request import urlopen, Request from urllib.request import urlopen, Request
import warnings
import pytest import pytest
...@@ -45,6 +46,31 @@ def limit_requests_per_time(min_secs_between_requests=2.0): ...@@ -45,6 +46,31 @@ def limit_requests_per_time(min_secs_between_requests=2.0):
urlopen = limit_requests_per_time()(urlopen) urlopen = limit_requests_per_time()(urlopen)
def resolve_redirects(max_redirects=3):
def outer_wrapper(fn):
def inner_wrapper(request, *args, **kwargs):
url = initial_url = request.full_url if isinstance(request, Request) else request
for _ in range(max_redirects + 1):
response = fn(request, *args, **kwargs)
if response.url == url or response.url is None:
if url != initial_url:
warnings.warn(f"The URL {initial_url} ultimately redirects to {url}.")
return response
url = response.url
else:
raise RecursionError(f"Request to {initial_url} exceeded {max_redirects} redirects.")
return inner_wrapper
return outer_wrapper
urlopen = resolve_redirects()(urlopen)
@contextlib.contextmanager @contextlib.contextmanager
def log_download_attempts( def log_download_attempts(
urls_and_md5s=None, urls_and_md5s=None,
...@@ -117,19 +143,22 @@ def assert_server_response_ok(): ...@@ -117,19 +143,22 @@ def assert_server_response_ok():
raise AssertionError("The request timed out.") from error raise AssertionError("The request timed out.") from error
except HTTPError as error: except HTTPError as error:
raise AssertionError(f"The server returned {error.code}: {error.reason}.") from error raise AssertionError(f"The server returned {error.code}: {error.reason}.") from error
except RecursionError as error:
raise AssertionError(str(error)) from error
def assert_url_is_accessible(url): def assert_url_is_accessible(url, timeout=5.0):
request = Request(url, headers=dict(method="HEAD")) request = Request(url, headers=dict(method="HEAD"))
with assert_server_response_ok(): with assert_server_response_ok():
urlopen(request, timeout=5.0) urlopen(request, timeout=timeout)
def assert_file_downloads_correctly(url, md5): def assert_file_downloads_correctly(url, md5, timeout=5.0):
with get_tmp_dir() as root: with get_tmp_dir() as root:
file = path.join(root, path.basename(url)) file = path.join(root, path.basename(url))
with assert_server_response_ok(): with assert_server_response_ok():
with urlopen(url, timeout=5.0) as response, open(file, "wb") as fh: with open(file, "wb") as fh:
response = urlopen(url, timeout=timeout)
fh.write(response.read()) fh.write(response.read())
assert check_integrity(file, md5=md5), "The MD5 checksums mismatch" assert check_integrity(file, md5=md5), "The MD5 checksums mismatch"
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment