test page

from pathlib import Path from time import process_time from urllib.parse import urlparse import pytest from scrapy.http import HtmlResponse, Response, TextResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import ( _remove_html_comments, get_base_url, get_meta_refresh, open_in_browser, response_status_message, ) def _read_browser_output(burl: str): path = urlparse(burl).path if not path or not Path(path).exists(): path = burl.replace("file://", "") return Path(path).read_bytes() def test_open_in_browser(): url = "http://www.example.com/some/page.html" body = ( b" test page test body " ) def browser_open(burl: str) -> bool: bbody = _read_browser_output(burl) assert b'' in bbody return True response = HtmlResponse(url, body=body) assert open_in_browser(response, _openfunc=browser_open), "Browser not called" resp = Response(url, body=body) with pytest.raises(TypeError): open_in_browser(resp, _openfunc=browser_open) # type: ignore[arg-type] def test_get_meta_refresh(): r1 = HtmlResponse( "http://www.example.com", body=b""" Dummy blahablsdfsal& """, ) r2 = HtmlResponse( "http://www.example.com", body=b""" Dummy blahablsdfsal& """, ) r3 = HtmlResponse( "http://www.example.com", body=b""" """, ) r4 = HtmlResponse( "http://www.example.com", body=b""" Dummy blahablsdfsal& """, ) assert get_meta_refresh(r1) == (5.0, "http://example.org/newpage") assert get_meta_refresh(r2) == (None, None) assert get_meta_refresh(r3) == (None, None) assert get_meta_refresh(r4) == ( 5.0, "http://www.another-domain.com/base/path/target.html", ) def test_get_base_url(): resp = HtmlResponse( "http://www.example.com", body=b""" blahablsdfsal& """, ) assert get_base_url(resp) == "http://www.example.com/img/" resp2 = HtmlResponse( "http://www.example.com", body=b""" blahablsdfsal&""", ) assert get_base_url(resp2) == "http://www.example.com" def test_response_status_message(): assert response_status_message(200) == "200 OK" assert response_status_message(404) == "404 Not Found" assert response_status_message(573) == "573 Unknown Status" @pytest.mark.parametrize( "body", [ pytest.param( b""" Dummy

Hello world.

""", id="Simple", ), pytest.param( b""" Dummy Hello world. """, id=" with attrs", ), pytest.param( b""" Dummy

Hello header

Hello world.

""", id="Misleading tag", ), pytest.param( b""" Dummy

Hello world.

""", id="Misleading comment", ), pytest.param( b""" Standard head

Hello world.

""", id="Conditional comment", ), ], ) def test_inject_base_url(body: bytes) -> None: url = "http://www.example.com" def check_base_url(burl): bbody = _read_browser_output(burl) assert bbody.count(b'>') == 1 assert b"/ (old pattern to remove comments). body = b"->" response = HtmlResponse("https://example.com", body=body) start_time = process_time() open_in_browser(response, lambda url: True) end_time = process_time() assert end_time - start_time < MAX_CPU_TIME def test_open_in_browser_redos_head(): MAX_CPU_TIME = 0.02 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ # for /(|\s.*?>))/ (old pattern to find the head element). body = b"b", b"ab"), (b"ac", b"ac"), (b"acccd", b"acd"), (b"ad", b"ad"), (b"a -->b", b"a -->b"), (b"real", b"real"), ], ) def test_remove_html_comments(input_body, output_body): assert _remove_html_comments(input_body) == output_body def test_open_in_browser_preserves_html_comments(): url = "http://www.example.com" body = ( b"" b"" b"Real" b"content" b"" ) def check(burl): bbody = _read_browser_output(burl) assert b"" in bbody return True response = HtmlResponse(url, body=body) assert open_in_browser(response, _openfunc=check) def test_open_in_browser_does_not_inject_base_when_present(): url = "http://www.example.com" body = ( b"" b'T' b"hi" b"" ) def check(burl): bbody = _read_browser_output(burl) assert b'' not in bbody assert b'' in bbody return True response = HtmlResponse(url, body=body) assert open_in_browser(response, _openfunc=check) def test_open_in_browser_injects_base_when_only_in_comment(): url = "http://www.example.com" body = ( b"" b"" b"Real" b"content" b"" ) def check(burl): bbody = _read_browser_output(burl) assert b'' in bbody return True response = HtmlResponse(url, body=body) assert open_in_browser(response, _openfunc=check) def test_open_in_browser_injects_base_at_real_head_not_commented_head(): url = "http://www.example.com" body = ( b"" b"" b"Actual" b"hello" b"" ) def check(burl): bbody = _read_browser_output(burl) assert bbody.count(b'') == 1 base_pos = bbody.find(b'') title_pos = bbody.find(b"Actual") assert base_pos < title_pos return True response = HtmlResponse(url, body=body) assert open_in_browser(response, _openfunc=check) def test_open_in_browser_text_response_uses_txt_extension(): response = TextResponse("http://www.example.com", body=b"plain text content") def check(burl): assert burl.endswith(".txt") return True assert open_in_browser(response, _openfunc=check) def test_open_in_browser_raises_for_unsupported_response_type(): response = Response("http://www.example.com", body=b"binary") with pytest.raises(TypeError): open_in_browser(response, _openfunc=lambda _: True) # type: ignore[arg-type]