diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index cbbf201c4..869665406 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -2,6 +2,7 @@ import unittest import warnings +from itertools import chain from pathlib import Path from urllib.parse import urlparse @@ -26,6 +27,29 @@ from scrapy.utils.response import ( __doctests__ = ['scrapy.utils.response'] +# https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata +PRE_XTRACTMIME_HTML_STARTS = ( + b' bytes: + """Make odd bytes lowecase and even bytes uppercase. + + >>> crazy_case(b'foobar') + b'fOoBaR' + """ + return b"".join( + bytes([byte]).lower() if index % 2 == 0 else bytes([byte]).upper() + for index, byte in enumerate(value) + ) + # Scenarios that work the same with the previously-used, deprecated # scrapy.responsetypes.responsetypes.from_args @@ -230,6 +275,58 @@ PRE_XTRACTMIME_SCENARIOS = ( # https://codersblock.com/blog/the-smallest-valid-html5-page/ (b'\n.', HtmlResponse), + # https://mimesniff.spec.whatwg.org/#identifying-a-resource-with-an-unknown-mime-type + *( + (prefix + start + b">", HtmlResponse) + for prefix in ( + b"", + *(byte for byte in WHITESPACE_BYTES if byte != b"\x0c"), + ) + for start in ( + set_case(start) + for set_case in (bytes.lower, bytes.upper, crazy_case) + for start in PRE_XTRACTMIME_HTML_STARTS + ) + ), + *( + (prefix + b"", HtmlResponse) + for start in ( + set_case(start) + for set_case in (bytes.lower, bytes.upper, crazy_case) + for start in POST_XTRACTMIME_HTML_STARTS + ) + ), + *( + (start + b" ", HtmlResponse) + for start in ( + set_case(start) + for set_case in (bytes.lower, bytes.upper, crazy_case) + for start in chain( + PRE_XTRACTMIME_HTML_STARTS, + POST_XTRACTMIME_HTML_STARTS, + ) + ) + ), + *( + (b"\x0c" + start + b">", HtmlResponse) + for start in ( + set_case(start) + for set_case in (bytes.lower, bytes.upper, crazy_case) + for start in PRE_XTRACTMIME_HTML_STARTS + ) + ), + (b"\x0c', TextResponse),