Speed up tests, remove comments without regexps

.github/workflows/tests-ubuntu.yml

@@ -50,9 +50,6 @@ jobs:
         - python-version: "3.12"
           env:
             TOXENV: botocore
-        - python-version: "3.12"
-          env:
-            TOXENV: slow
     steps:
     - uses: actions/checkout@v3
 

scrapy/utils/response.py

@@ -74,6 +74,18 @@ def response_httprepr(response: Response) -> bytes:
     return b"".join(values)
 
 
+def _remove_html_comments(body):
+    start = body.find(b"<!--")
+    while start != -1:
+        end = body.find(b"-->", start + 1)
+        if end == -1:
+            return body[:start]
+        else:
+            body = body[:start] + body[end + 3 :]
+            start = body.find(b"<!--")
+    return body
+
+
 def open_in_browser(
     response: Union[
         "scrapy.http.response.html.HtmlResponse",
@@ -103,8 +115,8 @@ def open_in_browser(
     body = response.body
     if isinstance(response, HtmlResponse):
         if b"<base" not in body:
+            _remove_html_comments(body)
             repl = rf'\0<base href="{response.url}">'
-            body = re.sub(b"(?s)<!--.*?(?:-->|$)", b"", body)
             body = re.sub(rb"<head(?:[^<>]*?>)", to_bytes(repl), body, count=1)
         ext = ".html"
     elif isinstance(response, TextResponse):

tests/test_utils_response.py

@@ -8,9 +8,9 @@ import pytest
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import HtmlResponse, Response, TextResponse
-from scrapy.settings.default_settings import DOWNLOAD_MAXSIZE
 from scrapy.utils.python import to_bytes
 from scrapy.utils.response import (
+    _remove_html_comments,
     get_base_url,
     get_meta_refresh,
     open_in_browser,
@@ -203,14 +203,13 @@ class ResponseUtilsTest(unittest.TestCase):
             r5, _openfunc=check_base_url
         ), "Inject unique base url with conditional comment"
 
-    @pytest.mark.slow
     def test_open_in_browser_redos_comment(self):
-        MAX_CPU_TIME = 30
+        MAX_CPU_TIME = 0.001
 
         # Exploit input from
         # https://makenowjust-labs.github.io/recheck/playground/
         # for /<!--.*?-->/ (old pattern to remove comments).
-        body = b"-><!--\x00" * (int(DOWNLOAD_MAXSIZE / 7) - 10) + b"->\n<!---->"
+        body = b"-><!--\x00" * 25_000 + b"->\n<!---->"
 
         response = HtmlResponse("https://example.com", body=body)
 
@@ -221,14 +220,13 @@ class ResponseUtilsTest(unittest.TestCase):
         end_time = process_time()
         self.assertLess(end_time - start_time, MAX_CPU_TIME)
 
-    @pytest.mark.slow
     def test_open_in_browser_redos_head(self):
-        MAX_CPU_TIME = 15
+        MAX_CPU_TIME = 0.001
 
         # Exploit input from
         # https://makenowjust-labs.github.io/recheck/playground/
         # for /(<head(?:>|\s.*?>))/ (old pattern to find the head element).
-        body = b"<head\t" * int(DOWNLOAD_MAXSIZE / 6)
+        body = b"<head\t" * 8_000
 
         response = HtmlResponse("https://example.com", body=body)
 
@@ -238,3 +236,42 @@ class ResponseUtilsTest(unittest.TestCase):
 
         end_time = process_time()
         self.assertLess(end_time - start_time, MAX_CPU_TIME)
+
+
+@pytest.mark.parametrize(
+    "input_body,output_body",
+    (
+        (
+            b"a<!--",
+            b"a",
+        ),
+        (
+            b"a<!---->b",
+            b"ab",
+        ),
+        (
+            b"a<!--b-->c",
+            b"ac",
+        ),
+        (
+            b"a<!--b-->c<!--",
+            b"ac",
+        ),
+        (
+            b"a<!--b-->c<!--d",
+            b"ac",
+        ),
+        (
+            b"a<!--b-->c<!---->d",
+            b"acd",
+        ),
+        (
+            b"a<!--b--><!--c-->d",
+            b"ad",
+        ),
+    ),
+)
+def test_remove_html_comments(input_body, output_body):
+    assert (
+        _remove_html_comments(input_body) == output_body
+    ), f"{_remove_html_comments(input_body)=} == {output_body=}"

tox.ini

@@ -221,9 +221,3 @@ setenv =
     {[pinned]setenv}
 commands =
     pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
-
-
-[testenv:slow]
-basepython = python3
-commands =
-    {[testenv]commands} -m 'slow'