diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 388ba9572..338c99584 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -50,9 +50,6 @@ jobs: - python-version: "3.12" env: TOXENV: botocore - - python-version: "3.12" - env: - TOXENV: slow steps: - uses: actions/checkout@v3 diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 4369e6439..fabfb1167 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -74,6 +74,18 @@ def response_httprepr(response: Response) -> bytes: return b"".join(values) +def _remove_html_comments(body): + start = body.find(b"", start + 1) + if end == -1: + return body[:start] + else: + body = body[:start] + body[end + 3 :] + start = body.find(b"|$)", b"", body) body = re.sub(rb"]*?>)", to_bytes(repl), body, count=1) ext = ".html" elif isinstance(response, TextResponse): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 1dbe187bf..db3c31b89 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -8,9 +8,9 @@ import pytest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response, TextResponse -from scrapy.settings.default_settings import DOWNLOAD_MAXSIZE from scrapy.utils.python import to_bytes from scrapy.utils.response import ( + _remove_html_comments, get_base_url, get_meta_refresh, open_in_browser, @@ -203,14 +203,13 @@ class ResponseUtilsTest(unittest.TestCase): r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" - @pytest.mark.slow def test_open_in_browser_redos_comment(self): - MAX_CPU_TIME = 30 + MAX_CPU_TIME = 0.001 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ # for // (old pattern to remove comments). - body = b"->" + body = b"->" response = HtmlResponse("https://example.com", body=body) @@ -221,14 +220,13 @@ class ResponseUtilsTest(unittest.TestCase): end_time = process_time() self.assertLess(end_time - start_time, MAX_CPU_TIME) - @pytest.mark.slow def test_open_in_browser_redos_head(self): - MAX_CPU_TIME = 15 + MAX_CPU_TIME = 0.001 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ # for /(|\s.*?>))/ (old pattern to find the head element). - body = b"b", + b"ab", + ), + ( + b"ac", + b"ac", + ), + ( + b"acccd", + b"acd", + ), + ( + b"ad", + b"ad", + ), + ), +) +def test_remove_html_comments(input_body, output_body): + assert ( + _remove_html_comments(input_body) == output_body + ), f"{_remove_html_comments(input_body)=} == {output_body=}" diff --git a/tox.ini b/tox.ini index e87d6a175..381da9773 100644 --- a/tox.ini +++ b/tox.ini @@ -221,9 +221,3 @@ setenv = {[pinned]setenv} commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} - - -[testenv:slow] -basepython = python3 -commands = - {[testenv]commands} -m 'slow'