test page

import unittest from pathlib import Path from time import process_time from urllib.parse import urlparse import pytest from scrapy.http import HtmlResponse, Response, TextResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import ( _remove_html_comments, get_base_url, get_meta_refresh, open_in_browser, response_status_message, ) __doctests__ = ["scrapy.utils.response"] class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url="http://example.org/", body=b"dummy_response") def test_open_in_browser(self): url = "http:///www.example.com/some/page.html" body = b" test page test body " def browser_open(burl): path = urlparse(burl).path if not path or not Path(path).exists(): path = burl.replace("file://", "") bbody = Path(path).read_bytes() self.assertIn(b'', bbody) return True response = HtmlResponse(url, body=body) assert open_in_browser(response, _openfunc=browser_open), "Browser not called" resp = Response(url, body=body) self.assertRaises(TypeError, open_in_browser, resp, debug=True) def test_get_meta_refresh(self): r1 = HtmlResponse( "http://www.example.com", body=b""" Dummy blahablsdfsal& """, ) r2 = HtmlResponse( "http://www.example.com", body=b""" Dummy blahablsdfsal& """, ) r3 = HtmlResponse( "http://www.example.com", body=b""" """, ) self.assertEqual(get_meta_refresh(r1), (5.0, "http://example.org/newpage")) self.assertEqual(get_meta_refresh(r2), (None, None)) self.assertEqual(get_meta_refresh(r3), (None, None)) def test_get_base_url(self): resp = HtmlResponse( "http://www.example.com", body=b""" blahablsdfsal& """, ) self.assertEqual(get_base_url(resp), "http://www.example.com/img/") resp2 = HtmlResponse( "http://www.example.com", body=b""" blahablsdfsal&""", ) self.assertEqual(get_base_url(resp2), "http://www.example.com") def test_response_status_message(self): self.assertEqual(response_status_message(200), "200 OK") self.assertEqual(response_status_message(404), "404 Not Found") self.assertEqual(response_status_message(573), "573 Unknown Status") def test_inject_base_url(self): url = "http://www.example.com" def check_base_url(burl): path = urlparse(burl).path if not path or not Path(path).exists(): path = burl.replace("file://", "") bbody = Path(path).read_bytes() self.assertEqual(bbody.count(b''), 1) return True r1 = HtmlResponse( url, body=b""" Dummy

Hello world.

""", ) r2 = HtmlResponse( url, body=b""" Dummy Hello world. """, ) r3 = HtmlResponse( url, body=b""" Dummy

Hello header

Hello world.

""", ) r4 = HtmlResponse( url, body=b""" Dummy

Hello world.

""", ) r5 = HtmlResponse( url, body=b""" Standard head

Hello world.

""", ) assert open_in_browser(r1, _openfunc=check_base_url), "Inject base url" assert open_in_browser(r2, _openfunc=check_base_url), ( "Inject base url with argumented head" ) assert open_in_browser(r3, _openfunc=check_base_url), ( "Inject unique base url with misleading tag" ) assert open_in_browser(r4, _openfunc=check_base_url), ( "Inject unique base url with misleading comment" ) assert open_in_browser(r5, _openfunc=check_base_url), ( "Inject unique base url with conditional comment" ) def test_open_in_browser_redos_comment(self): MAX_CPU_TIME = 0.02 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ # for // (old pattern to remove comments). body = b"->" response = HtmlResponse("https://example.com", body=body) start_time = process_time() open_in_browser(response, lambda url: True) end_time = process_time() self.assertLess(end_time - start_time, MAX_CPU_TIME) def test_open_in_browser_redos_head(self): MAX_CPU_TIME = 0.02 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ # for /(|\s.*?>))/ (old pattern to find the head element). body = b"b", b"ab", ), ( b"ac", b"ac", ), ( b"acccd", b"acd", ), ( b"ad", b"ad", ), ), ) def test_remove_html_comments(input_body, output_body): assert _remove_html_comments(input_body) == output_body, ( f"{_remove_html_comments(input_body)=} == {output_body=}" )