Hello

from itertools import chain from pathlib import Path from time import process_time from urllib.parse import urlparse import pytest from xtractmime import BINARY_BYTES, RESOURCE_HEADER_BUFFER_LENGTH from scrapy.http import HtmlResponse, JsonResponse, Response, TextResponse, XmlResponse from scrapy.http.headers import Headers from scrapy.responsetypes import ResponseTypes from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes from scrapy.utils.response import ( _get_encoding_or_mime_type_from_headers, _remove_html_comments, get_base_url, get_meta_refresh, get_response_class, open_in_browser, response_status_message, ) def _read_browser_output(burl: str): path = urlparse(burl).path if not path or not Path(path).exists(): path = burl.replace("file://", "") return Path(path).read_bytes() # https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata PRE_XTRACTMIME_HTML_STARTS = ( b" bytes: """Make odd bytes lowecase and even bytes uppercase. >>> odd_capitalize(b'foobar') b'fOoBaR' """ return b"".join( bytes([byte]).lower() if index % 2 == 0 else bytes([byte]).upper() for index, byte in enumerate(value) ) # Scenarios that work the same with the previously-used, deprecated # scrapy.responsetypes.responsetypes.from_args PRE_XTRACTMIME_SCENARIOS = ( # Content-Type determines the type for the HTTP protocol. *( ( { "url": f"{protocol}://example.com/foo", "headers": Headers( {"Content-Type": content_type + content_type_parameters} ), }, response_class, ) for protocol in ("http", "https") # Make sure that MIME parameters do not break response class choice. for content_type_parameters in ("", "; foo=bar") for content_type, response_class in ( ("text/plain", TextResponse), ("text/html", HtmlResponse), ("text/xml", XmlResponse), *( (mime_type, load_object(class_path)) for mime_type, class_path in ResponseTypes.CLASSES.items() if mime_type not in ( # “Note that XHTML is best parsed as XML” # https://lxml.de/parsing.html "application/xhtml+xml", "application/vnd.wap.xhtml+xml", ) ), # JavaScript MIME types should trigger a TextResponse. # # https://mimesniff.spec.whatwg.org/#javascript-mime-type *( (mime_type, TextResponse) for mime_type in ( "application/javascript", "application/x-javascript", "text/ecmascript", "text/javascript", "text/javascript1.0", "text/javascript1.1", "text/javascript1.2", "text/javascript1.3", "text/javascript1.4", "text/javascript1.5", "text/jscript", "text/livescript", "text/x-ecmascript", "text/x-javascript", # Unofficial "application/x-javascript", ) ), # JSON MIME types should trigger a JsonResponse. # # https://mimesniff.spec.whatwg.org/#json-mime-type *( (mime_type, JsonResponse) for mime_type in ( "application/json", # Unofficial "application/json-amazonui-streaming", "application/x-json", ) ), ) ), # Content-Type triumphs body, except for: # # - Binary content mislabeled as plain text due to an Apache bug # https://mimesniff.spec.whatwg.org/#check-for-apache-bug-flag # https://mimesniff.spec.whatwg.org/#rules-for-text-or-binary # # - Feeds mislabeled as HTML # https://mimesniff.spec.whatwg.org/#rules-for-distinguishing-if-a-resource-is-a-feed-or-html *( ( { "body": body, "headers": Headers({"Content-Type": [content_type]}), }, response_class, ) for body, content_type, response_class in ( *( (b"\x00\x01\xff", content_type, TextResponse) for content_type in ( # text/plain variants *not* affected by the Apache bug "text/plain; charset=Iso-8859-1", "text/plain; charset=utf-8", "text/plain; charset=windows-1252", ) ), ) ), # Content-Type triumphs Content-Disposition. *( ( { "url": f"{protocol}://example.com/a", "headers": Headers( { "Content-Disposition": [ f'attachment; filename="a.{file_extension}"', ], "Content-Type": [content_type], } ), }, response_class, ) for protocol in ("http", "https") for file_extension, content_type, response_class in ( ("html", "application/json", JsonResponse), ("xml", "application/json", JsonResponse), ) ), # Compressed content should be of type Response until uncompressed. ( { "headers": Headers( { "Content-Encoding": ["zip"], "Content-Type": ["text/html"], } ) }, Response, ), # We take the file extension of URL paths into account, except for HTTP # responses, because “they are unreliable and easily spoofed”. # # https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata *( ( {"url": f"{protocol}://example.com/a.{extension}"}, response_class, ) for protocol in ("file", "ftp") for extension, response_class in ( ("gz", Response), ("html", HtmlResponse), ("json", JsonResponse), ("pdf", Response), ("txt", TextResponse), ("xml", XmlResponse), ) ), # Unlike in a web browser, where an attachment Content-Disposition header # causes the response to be downloaded, and hence MIME sniffing becomes # irrelevant, in Scrapy those responses are handled the same as any, and # hence we take the file extension from Content-Disposition into account # to choose a response class, as a fallback when there is no Content-Type. *( ( { "url": f"{protocol}://example.com/a", "headers": Headers( { "Content-Disposition": [ 'attachment; filename="a.xml"', ] } ), }, XmlResponse, ) for protocol in ("http", "https") ), *( ( { "url": f"{protocol}://example.com/a", "headers": Headers( { "Content-Disposition": [ 'attachment; filename="a.html"', ], "Content-Type": "text/xml", } ), }, XmlResponse, ) for protocol in ("http", "https") ), *( ( { "url": f"{protocol}://example.com/a", "body": b"Hello", HtmlResponse), (b'\n.", HtmlResponse), # https://mimesniff.spec.whatwg.org/#identifying-a-resource-with-an-unknown-mime-type *( (prefix + start + b">", HtmlResponse) for prefix in ( b"", *(byte for byte in WHITESPACE_BYTES if byte != b"\x0c"), ) for start in ( set_case(start) for set_case in (bytes.lower, bytes.upper, odd_capitalize) for start in PRE_XTRACTMIME_HTML_STARTS ) ), *( (prefix + b"", "headers": Headers( { "Content-Encoding": ["zip"], } ), }, Response, ), # If the body is empty, it contains no binary data bytes, hence body-based # MIME type detection must interpret the result as text. # # https://mimesniff.spec.whatwg.org/#identifying-a-resource-with-an-unknown-mime-type ({}, TextResponse), # We take the file extension of URL paths into account, except for HTTP # responses, because “they are unreliable and easily spoofed”. # # https://mimesniff.spec.whatwg.org/#interpreting-the-resource-metadata *( ( {"url": f"{protocol}://example.com/a.{extension}"}, response_class, ) for protocol in ("file", "ftp") for extension, response_class in ( # “Note that XHTML is best parsed as XML” # https://lxml.de/parsing.html ("xhtml", XmlResponse), ) ), *( ( {"url": f"{protocol}://example.com/a.html"}, response_class, ) for protocol, response_class in ( *((protocol, TextResponse) for protocol in ("http", "https")), ) ), # File extension triumphs body. ( { "body": b"", "headers": Headers( { "Content-Disposition": [ 'attachment; filename="a.gz"', ], } ), }, Response, ), ( { "body": b"", "url": "file:///a.gz", }, Response, ), # Without anything else, the body determines the response class. *( ({"body": body}, response_class) for body, response_class in ( # https://mimesniff.spec.whatwg.org/#identifying-a-resource-with-an-unknown-mime-type *( (start + b">", HtmlResponse) for start in ( set_case(start) for set_case in (bytes.lower, bytes.upper, odd_capitalize) for start in POST_XTRACTMIME_HTML_STARTS ) ), *( (start + b" ", HtmlResponse) for start in ( set_case(start) for set_case in (bytes.lower, bytes.upper, odd_capitalize) for start in chain( PRE_XTRACTMIME_HTML_STARTS, POST_XTRACTMIME_HTML_STARTS, ) ) ), *( (b"\x0c" + start + b">", HtmlResponse) for start in ( set_case(start) for set_case in (bytes.lower, bytes.upper, odd_capitalize) for start in PRE_XTRACTMIME_HTML_STARTS ) ), (b"\x0c", TextResponse), (b"a test page test body " ) def browser_open(burl: str) -> bool: bbody = _read_browser_output(burl) assert b'' in bbody return True response = HtmlResponse(url, body=body) assert open_in_browser(response, _openfunc=browser_open), "Browser not called" resp = Response(url, body=body) with pytest.raises(TypeError): open_in_browser(resp, debug=True) # pylint: disable=unexpected-keyword-arg def test_get_meta_refresh(): r1 = HtmlResponse( "http://www.example.com", body=b""" Dummy blahablsdfsal& """, ) r2 = HtmlResponse( "http://www.example.com", body=b""" Dummy blahablsdfsal& """, ) r3 = HtmlResponse( "http://www.example.com", body=b""" """, ) r4 = HtmlResponse( "http://www.example.com", body=b""" Dummy blahablsdfsal& """, ) assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage") assert get_meta_refresh(r2) == (None, None) assert get_meta_refresh(r3) == (None, None) assert get_meta_refresh(r4) == ( 5.0, "http://www.another-domain.com/base/path/target.html", ) def test_get_base_url(): resp = HtmlResponse( "http://www.example.com", body=b""" blahablsdfsal& """, ) assert get_base_url(resp) == "http://www.example.com/img/" resp2 = HtmlResponse( "http://www.example.com", body=b""" blahablsdfsal&""", ) assert get_base_url(resp2) == "http://www.example.com" def test_response_status_message(): assert response_status_message(200) == "200 OK" assert response_status_message(404) == "404 Not Found" assert response_status_message(573) == "573 Unknown Status" @pytest.mark.parametrize( "body", [ pytest.param( b""" Dummy

Hello world.

""", id="Simple", ), pytest.param( b""" Dummy Hello world. """, id=" with attrs", ), pytest.param( b""" Dummy

Hello header

Hello world.

""", id="Misleading tag", ), pytest.param( b""" Dummy

Hello world.

""", id="Misleading comment", ), pytest.param( b""" Standard head

Hello world.

""", id="Conditional comment", ), ], ) def test_inject_base_url(body: bytes) -> None: url = "http://www.example.com" def check_base_url(burl): bbody = _read_browser_output(burl) assert bbody.count(b'>') == 1 assert b"/ (old pattern to remove comments). body = b"->" response = HtmlResponse("https://example.com", body=body) start_time = process_time() open_in_browser(response, lambda url: True) end_time = process_time() assert end_time - start_time < MAX_CPU_TIME def test_open_in_browser_redos_head(): MAX_CPU_TIME = 0.02 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ # for /(|\s.*?>))/ (old pattern to find the head element). body = b"b", b"ab"), (b"ac", b"ac"), (b"acccd", b"acd"), (b"ad", b"ad"), (b"a -->b", b"a -->b"), (b"real", b"real"), ], ) def test_remove_html_comments(input_body, output_body): assert _remove_html_comments(input_body) == output_body def test_open_in_browser_preserves_html_comments(): url = "http://www.example.com" body = ( b"" b"" b"Real" b"content" b"" ) def check(burl): bbody = _read_browser_output(burl) assert b"" in bbody return True response = HtmlResponse(url, body=body) assert open_in_browser(response, _openfunc=check) def test_open_in_browser_does_not_inject_base_when_present(): url = "http://www.example.com" body = ( b"" b'T' b"hi" b"" ) def check(burl): bbody = _read_browser_output(burl) assert b'' not in bbody assert b'' in bbody return True response = HtmlResponse(url, body=body) assert open_in_browser(response, _openfunc=check) def test_open_in_browser_injects_base_when_only_in_comment(): url = "http://www.example.com" body = ( b"" b"" b"Real" b"content" b"" ) def check(burl): bbody = _read_browser_output(burl) assert b'' in bbody return True response = HtmlResponse(url, body=body) assert open_in_browser(response, _openfunc=check) def test_open_in_browser_injects_base_at_real_head_not_commented_head(): url = "http://www.example.com" body = ( b"" b"" b"Actual" b"hello" b"" ) def check(burl): bbody = _read_browser_output(burl) assert bbody.count(b'') == 1 base_pos = bbody.find(b'') title_pos = bbody.find(b"Actual") assert base_pos < title_pos return True response = HtmlResponse(url, body=body) assert open_in_browser(response, _openfunc=check) def test_open_in_browser_text_response_uses_txt_extension(): response = TextResponse("http://www.example.com", body=b"plain text content") def check(burl): assert burl.endswith(".txt") return True assert open_in_browser(response, _openfunc=check) def test_open_in_browser_raises_for_unsupported_response_type(): response = Response("http://www.example.com", body=b"binary") with pytest.raises(TypeError): open_in_browser(response, _openfunc=lambda _: True)