From c947f51077e1ab246f30764cc4cc7a1cc5835d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 29 Nov 2023 11:54:08 +0100 Subject: [PATCH 01/18] Set an arbitrary upper limit on ReDoS-vulnerable regexps --- docs/news.rst | 28 ++++++++++++++++++++++++++++ scrapy/utils/iterators.py | 12 +++++++----- scrapy/utils/response.py | 4 ++-- 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index fd8fa3ea3..121cc0322 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,22 @@ Release notes ============= +.. _release-2.11.1: + +Scrapy 2.11.1 (unreleased) +-------------------------- + +**Security bug fix:** + +- The regular expressions of the ``iternodes`` node iterator of + :class:`~scrapy.spiders.XMLFeedSpider` are no longer susceptible to a + `ReDoS attack`_. Please, see the `cc65-xxvf-f7r9 security + advisory`_ for more information. + + .. _ReDoS attack: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS + .. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9 + + .. _release-2.11.0: Scrapy 2.11.0 (2023-09-18) @@ -2869,6 +2885,18 @@ affect subclasses: (:issue:`3884`) +.. _release-1.8.4: + +Scrapy 1.8.4 (unreleased) +------------------------- + +**Security bug fix:** + +- The regular expressions of the ``iternodes`` node iterator of + :class:`~scrapy.spiders.XMLFeedSpider` are no longer susceptible to a + `ReDoS attack`_. Please, see the `cc65-xxvf-f7r9 security + advisory`_ for more information. + .. _release-1.8.3: diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 03d779afb..6b89334e0 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -40,10 +40,10 @@ def xmliter( """ nodename_patt = re.escape(nodename) - DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) + DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]{1,1024}>\s*", re.S) HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S) - END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S) - NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) + END_TAG_RE = re.compile(r"<\s*/([^\s>]{1,1024})\s*>", re.S) + NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]{,1024})=[^>\s]+)", re.S) text = _body_or_str(obj) document_header_match = re.search(DOCUMENT_HEADER_RE, text) @@ -57,13 +57,15 @@ def xmliter( for tagname in reversed(re.findall(END_TAG_RE, header_end)): assert header_end_idx tag = re.search( - rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S + rf"<\s*{tagname}.{{,1024}}?xmlns[:=][^>]{{,1024}}>", + text[: header_end_idx[1]], + re.S, ) if tag: for x in re.findall(NAMESPACE_RE, tag.group()): namespaces[x[1]] = x[0] - r = re.compile(rf"<{nodename_patt}[\s>].*?", re.DOTALL) + r = re.compile(rf"<{nodename_patt}[\s>].{{,1024}}?", re.DOTALL) for match in r.finditer(text): nodetext = ( document_header diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index c540d6278..b0e106c5e 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -91,8 +91,8 @@ def open_in_browser( if isinstance(response, HtmlResponse): if b"' - body = re.sub(b"", b"", body, flags=re.DOTALL) - body = re.sub(rb"(|\s.*?>))", to_bytes(repl), body) + body = re.sub(b"", b"", body, flags=re.DOTALL) + body = re.sub(rb"(|\s.{,1024}?>))", to_bytes(repl), body) ext = ".html" elif isinstance(response, TextResponse): ext = ".txt" From eb8b2c5197f3dd8570bad1945b23886ee57d045f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 29 Nov 2023 12:13:04 +0100 Subject: [PATCH 02/18] Mention open_in_browser in the release notes --- docs/news.rst | 16 ++++++++-------- docs/topics/debug.rst | 18 +++--------------- scrapy/utils/response.py | 17 +++++++++++++++-- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 121cc0322..2bbe833a5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -10,10 +10,10 @@ Scrapy 2.11.1 (unreleased) **Security bug fix:** -- The regular expressions of the ``iternodes`` node iterator of - :class:`~scrapy.spiders.XMLFeedSpider` are no longer susceptible to a - `ReDoS attack`_. Please, see the `cc65-xxvf-f7r9 security - advisory`_ for more information. +- Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the + ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider` and + the :func:`~scrapy.utils.response.open_in_browser` function. Please, see + the `cc65-xxvf-f7r9 security advisory`_ for more information. .. _ReDoS attack: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS .. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9 @@ -2892,10 +2892,10 @@ Scrapy 1.8.4 (unreleased) **Security bug fix:** -- The regular expressions of the ``iternodes`` node iterator of - :class:`~scrapy.spiders.XMLFeedSpider` are no longer susceptible to a - `ReDoS attack`_. Please, see the `cc65-xxvf-f7r9 security - advisory`_ for more information. +- Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the + ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider` and + the :func:`~scrapy.utils.response.open_in_browser` function. Please, see + the `cc65-xxvf-f7r9 security advisory`_ for more information. .. _release-1.8.3: diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 49c5b0410..988e37bbd 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -125,25 +125,15 @@ Fortunately, the :command:`shell` is your bread and butter in this case (see See also: :ref:`topics-shell-inspect-response`. + Open in browser =============== Sometimes you just want to see how a certain response looks in a browser, you -can use the ``open_in_browser`` function for that. Here is an example of how -you would use it: +can use the :func:`~scrapy.utils.response.open_in_browser` function for that: -.. code-block:: python +.. autofunction:: scrapy.utils.response.open_in_browser - from scrapy.utils.response import open_in_browser - - - def parse_details(self, response): - if "item name" not in response.body: - open_in_browser(response) - -``open_in_browser`` will open a browser with the response received by Scrapy at -that point, adjusting the `base tag`_ so that images and styles are displayed -properly. Logging ======= @@ -163,8 +153,6 @@ available in all future runs should they be necessary again: For more information, check the :ref:`topics-logging` section. -.. _base tag: https://www.w3schools.com/tags/tag_base.asp - .. _debug-vscode: Visual Studio Code diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index b0e106c5e..8401d4ed1 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -81,8 +81,21 @@ def open_in_browser( ], _openfunc: Callable[[str], Any] = webbrowser.open, ) -> Any: - """Open the given response in a local web browser, populating the - tag for external links to work + """Open *response* in a local web browser, adjusting the `base tag`_ for + external links to work, e.g. so that images and styles are displayed. + + .. _base tag: https://www.w3schools.com/tags/tag_base.asp + + For example: + + .. code-block:: python + + from scrapy.utils.response import open_in_browser + + + def parse_details(self, response): + if "item name" not in response.body: + open_in_browser(response) """ from scrapy.http import HtmlResponse, TextResponse From 40b3efbbee3919e8f6900daeaed5e14183cabfd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 29 Nov 2023 12:47:04 +0100 Subject: [PATCH 03/18] Remove open_in_browser from the 1.8.4 release notes --- docs/news.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 2bbe833a5..c14815d06 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -2893,9 +2893,8 @@ Scrapy 1.8.4 (unreleased) **Security bug fix:** - Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the - ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider` and - the :func:`~scrapy.utils.response.open_in_browser` function. Please, see - the `cc65-xxvf-f7r9 security advisory`_ for more information. + ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider`. + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. .. _release-1.8.3: From 1533b69032e2fb5e495e88a3fed57c0d98502612 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 13 Dec 2023 12:01:35 +0100 Subject: [PATCH 04/18] Test and address ReDoS attack vectors for open_in_browser --- scrapy/utils/response.py | 6 +++--- tests/test_utils_response.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 8401d4ed1..4369e6439 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -103,9 +103,9 @@ def open_in_browser( body = response.body if isinstance(response, HtmlResponse): if b"' - body = re.sub(b"", b"", body, flags=re.DOTALL) - body = re.sub(rb"(|\s.{,1024}?>))", to_bytes(repl), body) + repl = rf'\0' + body = re.sub(b"(?s)|$)", b"", body) + body = re.sub(rb"]*?>)", to_bytes(repl), body, count=1) ext = ".html" elif isinstance(response, TextResponse): ext = ".txt" diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 80e15a60f..942584d92 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -1,10 +1,12 @@ import unittest import warnings from pathlib import Path +from time import process_time from urllib.parse import urlparse from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response, TextResponse +from scrapy.settings.default_settings import DOWNLOAD_MAXSIZE from scrapy.utils.python import to_bytes from scrapy.utils.response import ( get_base_url, @@ -198,3 +200,37 @@ class ResponseUtilsTest(unittest.TestCase): assert open_in_browser( r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" + + def test_open_in_browser_redos_comment(self): + MAX_CPU_TIME = 30 + + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for // (old pattern to remove comments). + body = b"->" + + response = HtmlResponse("https://example.com", body=body) + + start_time = process_time() + + open_in_browser(response, lambda url: True) + + end_time = process_time() + self.assertLess(end_time - start_time, MAX_CPU_TIME) + + def test_open_in_browser_redos_head(self): + MAX_CPU_TIME = 15 + + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for /(|\s.*?>))/ (old pattern to find the head element). + body = b" Date: Fri, 15 Dec 2023 10:06:13 +0100 Subject: [PATCH 05/18] Fix namespaces nodename support for xmliter_lxml --- scrapy/utils/iterators.py | 23 +++++++++++++++++++++-- tests/test_utils_iterators.py | 5 ----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 6b89334e0..9c53ab524 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -12,11 +12,14 @@ from typing import ( List, Literal, Optional, + Tuple, Union, cast, overload, ) +from lxml import etree + from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode @@ -77,15 +80,31 @@ def xmliter( yield Selector(text=nodetext, type="xml") +def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: + if ":" not in element_name: + return element_name, None, None + reader: "SupportsReadClose[bytes]" = _StreamReader(data) + node_prefix, element_name = element_name.split(":", maxsplit=1) + ns_iterator = etree.iterparse( + reader, encoding=reader.encoding, events=("start-ns",) + ) + for event, (_prefix, _namespace) in ns_iterator: + if _prefix != node_prefix: + continue + return element_name, _prefix, _namespace + return f"{node_prefix}:{element_name}", None, None + + def xmliter_lxml( obj: Union[Response, str, bytes], nodename: str, namespace: Optional[str] = None, prefix: str = "x", ) -> Generator[Selector, Any, None]: - from lxml import etree + if not namespace: + nodename, prefix, namespace = _resolve_xml_namespace(nodename, obj) - reader = _StreamReader(obj) + reader: "SupportsReadClose[bytes]" = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 3598fa0bb..24f03155b 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,4 +1,3 @@ -from pytest import mark from twisted.trial import unittest from scrapy.http import Response, TextResponse, XmlResponse @@ -247,10 +246,6 @@ class XmliterTestCase(unittest.TestCase): class LxmlXmliterTestCase(XmliterTestCase): xmliter = staticmethod(xmliter_lxml) - @mark.xfail(reason="known bug of the current implementation") - def test_xmliter_namespaced_nodename(self): - super().test_xmliter_namespaced_nodename() - def test_xmliter_iterate_namespace(self): body = b""" From d50f436a73ef13fca8d3d9c302ae1a48e984a4a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 10:08:45 +0100 Subject: [PATCH 06/18] Enable huge_tree for xmliter_lxml --- scrapy/utils/iterators.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 9c53ab524..1c51c0c6a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -86,7 +86,10 @@ def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: reader: "SupportsReadClose[bytes]" = _StreamReader(data) node_prefix, element_name = element_name.split(":", maxsplit=1) ns_iterator = etree.iterparse( - reader, encoding=reader.encoding, events=("start-ns",) + reader, + encoding=reader.encoding, + events=("start-ns",), + huge_tree=True, ) for event, (_prefix, _namespace) in ns_iterator: if _prefix != node_prefix: @@ -107,7 +110,10 @@ def xmliter_lxml( reader: "SupportsReadClose[bytes]" = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( - cast("SupportsReadClose[bytes]", reader), tag=tag, encoding=reader.encoding + reader, + tag=tag, + encoding=reader.encoding, + huge_tree=True, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) for _, node in iterable: From 9655b0b8eb4bbc66b0fe540a19265b6342cf371b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 10:19:47 +0100 Subject: [PATCH 07/18] Mark slow tests, with their own tox env and CI job --- .github/workflows/tests-ubuntu.yml | 6 ++++ pytest.ini | 2 ++ tests/test_utils_response.py | 50 +++++++++++++++++------------- tox.ini | 6 ++++ 4 files changed, 42 insertions(+), 22 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 5ff92a571..7562cf22b 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -47,6 +47,9 @@ jobs: - python-version: "3.11" env: TOXENV: botocore + - python-version: "3.11" + env: + TOXENV: slow - python-version: "3.12.0-rc.2" env: @@ -57,6 +60,9 @@ jobs: - python-version: "3.12.0-rc.2" env: TOXENV: extra-deps + - python-version: "3.12.0-rc.2" + env: + TOXENV: slow steps: - uses: actions/checkout@v3 diff --git a/pytest.ini b/pytest.ini index 16983be5e..877fbcd1d 100644 --- a/pytest.ini +++ b/pytest.ini @@ -17,10 +17,12 @@ addopts = --ignore=docs/topics/stats.rst --ignore=docs/topics/telnetconsole.rst --ignore=docs/utils + -m 'not slow' markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed requires_uvloop: marks tests as only enabled when uvloop is known to be working + slow: marks tests as slow, not executed by default filterwarnings = ignore:scrapy.downloadermiddlewares.decompression is deprecated ignore:Module scrapy.utils.reqser is deprecated diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 942584d92..93b9bacaf 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -4,6 +4,8 @@ from pathlib import Path from time import process_time from urllib.parse import urlparse +import pytest + from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response, TextResponse from scrapy.settings.default_settings import DOWNLOAD_MAXSIZE @@ -201,36 +203,40 @@ class ResponseUtilsTest(unittest.TestCase): r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" - def test_open_in_browser_redos_comment(self): - MAX_CPU_TIME = 30 - # Exploit input from - # https://makenowjust-labs.github.io/recheck/playground/ - # for // (old pattern to remove comments). - body = b"->" +@pytest.mark.slow +def test_open_in_browser_redos_comment(): + MAX_CPU_TIME = 30 - response = HtmlResponse("https://example.com", body=body) + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for // (old pattern to remove comments). + body = b"->" - start_time = process_time() + response = HtmlResponse("https://example.com", body=body) - open_in_browser(response, lambda url: True) + start_time = process_time() - end_time = process_time() - self.assertLess(end_time - start_time, MAX_CPU_TIME) + open_in_browser(response, lambda url: True) - def test_open_in_browser_redos_head(self): - MAX_CPU_TIME = 15 + end_time = process_time() + assert (end_time - start_time) < MAX_CPU_TIME - # Exploit input from - # https://makenowjust-labs.github.io/recheck/playground/ - # for /(|\s.*?>))/ (old pattern to find the head element). - body = b"|\s.*?>))/ (old pattern to find the head element). + body = b" Date: Fri, 15 Dec 2023 10:23:24 +0100 Subject: [PATCH 08/18] Restore the implementation of xmliter --- scrapy/utils/iterators.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 1c51c0c6a..b67029433 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -43,10 +43,10 @@ def xmliter( """ nodename_patt = re.escape(nodename) - DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]{1,1024}>\s*", re.S) + DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.S) - END_TAG_RE = re.compile(r"<\s*/([^\s>]{1,1024})\s*>", re.S) - NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]{,1024})=[^>\s]+)", re.S) + END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.S) + NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.S) text = _body_or_str(obj) document_header_match = re.search(DOCUMENT_HEADER_RE, text) @@ -60,15 +60,13 @@ def xmliter( for tagname in reversed(re.findall(END_TAG_RE, header_end)): assert header_end_idx tag = re.search( - rf"<\s*{tagname}.{{,1024}}?xmlns[:=][^>]{{,1024}}>", - text[: header_end_idx[1]], - re.S, + rf"<\s*{tagname}.*?xmlns[:=][^>]*>", text[: header_end_idx[1]], re.S ) if tag: for x in re.findall(NAMESPACE_RE, tag.group()): namespaces[x[1]] = x[0] - r = re.compile(rf"<{nodename_patt}[\s>].{{,1024}}?", re.DOTALL) + r = re.compile(rf"<{nodename_patt}[\s>].*?", re.DOTALL) for match in r.finditer(text): nodetext = ( document_header From 150d96764b5a455c75315596ca8ba5ded0f416dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 11:42:55 +0100 Subject: [PATCH 09/18] Deprecate xmliter in favor of xmliter_lxml --- docs/faq.rst | 10 ++++--- docs/news.rst | 32 +++++++++++++++++----- scrapy/spiders/feed.py | 4 +-- scrapy/utils/iterators.py | 23 ++++++++++++++-- tests/test_utils_iterators.py | 35 +++++++++++++++++++++--- tests/test_utils_response.py | 50 +++++++++++++++++------------------ 6 files changed, 110 insertions(+), 44 deletions(-) diff --git a/docs/faq.rst b/docs/faq.rst index 20dd814df..657802fd3 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -297,9 +297,13 @@ build the DOM of the entire feed in memory, and this can be quite slow and consume a lot of memory. In order to avoid parsing all the entire feed at once in memory, you can use -the functions ``xmliter`` and ``csviter`` from ``scrapy.utils.iterators`` -module. In fact, this is what the feed spiders (see :ref:`topics-spiders`) use -under the cover. +the :func:`~scrapy.utils.iterators.xmliter_lxml` and +:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what +:class:`~scrapy.spiders.XMLFeedSpider` uses. + +.. autofunction:: scrapy.utils.iterators.xmliter_lxml + +.. autofunction:: scrapy.utils.iterators.csviter Does Scrapy manage cookies automatically? ----------------------------------------- diff --git a/docs/news.rst b/docs/news.rst index c14815d06..57b99e94f 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -10,12 +10,23 @@ Scrapy 2.11.1 (unreleased) **Security bug fix:** -- Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the - ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider` and - the :func:`~scrapy.utils.response.open_in_browser` function. Please, see - the `cc65-xxvf-f7r9 security advisory`_ for more information. +- Addressed `ReDoS vulnerabilities`_: - .. _ReDoS attack: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS + - ``scrapy.utils.iterators.xmliter`` is now deprecated in favor of + :func:`~scrapy.utils.iterators.xmliter_lxml`, which + :class:`~scrapy.spiders.XMLFeedSpider` now uses. + + To minimize the impact of this change on existing code, + :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating + the node namespace with a prefix in the node name, and big files with + highly nested trees. + + - Fixed regular expressions in the implementation of the + :func:`~scrapy.utils.response.open_in_browser` function. + + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. + + .. _ReDoS vulnerabilities: https://owasp.org/www-community/attacks/Regular_expression_Denial_of_Service_-_ReDoS .. _cc65-xxvf-f7r9 security advisory: https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9 @@ -2892,8 +2903,15 @@ Scrapy 1.8.4 (unreleased) **Security bug fix:** -- Fixed regular expressions susceptible to a `ReDoS attack`_ affecting the - ``iternodes`` node iterator of :class:`~scrapy.spiders.XMLFeedSpider`. +- Due to its `ReDoS vulnerabilities`_, ``scrapy.utils.iterators.xmliter`` is + now deprecated in favor of :func:`~scrapy.utils.iterators.xmliter_lxml`, + which :class:`~scrapy.spiders.XMLFeedSpider` now uses. + + To minimize the impact of this change on existing code, + :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating + the node namespace as a prefix in the node name, and big files with highly + nested trees when using lxml 4.2 or later. + Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py index 6afadc577..42675c76a 100644 --- a/scrapy/spiders/feed.py +++ b/scrapy/spiders/feed.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst from scrapy.exceptions import NotConfigured, NotSupported from scrapy.selector import Selector from scrapy.spiders import Spider -from scrapy.utils.iterators import csviter, xmliter +from scrapy.utils.iterators import csviter, xmliter_lxml from scrapy.utils.spider import iterate_spider_output @@ -84,7 +84,7 @@ class XMLFeedSpider(Spider): return self.parse_nodes(response, nodes) def _iternodes(self, response): - for node in xmliter(response, self.itertag): + for node in xmliter_lxml(response, self.itertag): self._register_namespaces(node) yield node diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index b67029433..7574e377a 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -17,9 +17,12 @@ from typing import ( cast, overload, ) +from warnings import warn from lxml import etree +from packaging.version import Version +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse from scrapy.selector import Selector from scrapy.utils.python import re_rsearch, to_unicode @@ -29,6 +32,12 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_LXML_VERSION = Version(etree.__version__) +_LXML_HUGE_TREE_VERSION = Version("4.2") +_ITERPARSE_KWARGS = {} +if _LXML_VERSION >= _LXML_HUGE_TREE_VERSION: + _ITERPARSE_KWARGS["huge_tree"] = True + def xmliter( obj: Union[Response, str, bytes], nodename: str @@ -41,6 +50,16 @@ def xmliter( - a unicode string - a string encoded as utf-8 """ + warn( + ( + "xmliter is deprecated and its use strongly discouraged because " + "it is vulnerable to ReDoS attacks. Use xmliter_lxml instead. See " + "https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9" + ), + ScrapyDeprecationWarning, + stacklevel=2, + ) + nodename_patt = re.escape(nodename) DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.S) @@ -87,7 +106,7 @@ def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: reader, encoding=reader.encoding, events=("start-ns",), - huge_tree=True, + **_ITERPARSE_KWARGS, ) for event, (_prefix, _namespace) in ns_iterator: if _prefix != node_prefix: @@ -111,7 +130,7 @@ def xmliter_lxml( reader, tag=tag, encoding=reader.encoding, - huge_tree=True, + **_ITERPARSE_KWARGS, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) for _, node in iterable: diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index 24f03155b..505cc276c 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -1,13 +1,14 @@ +import pytest from twisted.trial import unittest +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse, XmlResponse from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml from tests import get_testdata -class XmliterTestCase(unittest.TestCase): - xmliter = staticmethod(xmliter) - +class XmliterBaseTestCase: + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter(self): body = b""" @@ -39,6 +40,7 @@ class XmliterTestCase(unittest.TestCase): attrs, [("001", ["Name 1"], ["Type 1"]), ("002", ["Name 2"], ["Type 2"])] ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unusual_node(self): body = b""" @@ -52,6 +54,7 @@ class XmliterTestCase(unittest.TestCase): ] self.assertEqual(nodenames, [["matchme..."]]) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_unicode(self): # example taken from https://github.com/scrapy/scrapy/issues/1665 body = """ @@ -111,6 +114,7 @@ class XmliterTestCase(unittest.TestCase): [("26", ["-"], ["80"]), ("21", ["Ab"], ["76"]), ("27", ["A"], ["27"])], ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_text(self): body = ( '' @@ -122,6 +126,7 @@ class XmliterTestCase(unittest.TestCase): [["one"], ["two"]], ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaces(self): body = b""" @@ -161,6 +166,7 @@ class XmliterTestCase(unittest.TestCase): self.assertEqual(node.xpath("id/text()").getall(), []) self.assertEqual(node.xpath("price/text()").getall(), []) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename(self): body = b""" @@ -189,6 +195,7 @@ class XmliterTestCase(unittest.TestCase): ["http://www.mydummycompany.com/images/item1.jpg"], ) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_namespaced_nodename_missing(self): body = b""" @@ -213,6 +220,7 @@ class XmliterTestCase(unittest.TestCase): with self.assertRaises(StopIteration): next(my_iter) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_exception(self): body = ( '' @@ -225,10 +233,12 @@ class XmliterTestCase(unittest.TestCase): self.assertRaises(StopIteration, next, iter) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_objtype_exception(self): i = self.xmliter(42, "product") self.assertRaises(TypeError, next, i) + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_xmliter_encoding(self): body = ( b'\n' @@ -243,7 +253,24 @@ class XmliterTestCase(unittest.TestCase): ) -class LxmlXmliterTestCase(XmliterTestCase): +class XmliterTestCase(XmliterBaseTestCase, unittest.TestCase): + xmliter = staticmethod(xmliter) + + def test_deprecation(self): + body = b""" + + + + + """ + with pytest.warns( + ScrapyDeprecationWarning, + match="xmliter", + ): + next(self.xmliter(body, "product")) + + +class LxmlXmliterTestCase(XmliterBaseTestCase, unittest.TestCase): xmliter = staticmethod(xmliter_lxml) def test_xmliter_iterate_namespace(self): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 93b9bacaf..1dbe187bf 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -203,40 +203,38 @@ class ResponseUtilsTest(unittest.TestCase): r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" + @pytest.mark.slow + def test_open_in_browser_redos_comment(self): + MAX_CPU_TIME = 30 -@pytest.mark.slow -def test_open_in_browser_redos_comment(): - MAX_CPU_TIME = 30 + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for // (old pattern to remove comments). + body = b"->" - # Exploit input from - # https://makenowjust-labs.github.io/recheck/playground/ - # for // (old pattern to remove comments). - body = b"->" + response = HtmlResponse("https://example.com", body=body) - response = HtmlResponse("https://example.com", body=body) + start_time = process_time() - start_time = process_time() + open_in_browser(response, lambda url: True) - open_in_browser(response, lambda url: True) + end_time = process_time() + self.assertLess(end_time - start_time, MAX_CPU_TIME) - end_time = process_time() - assert (end_time - start_time) < MAX_CPU_TIME + @pytest.mark.slow + def test_open_in_browser_redos_head(self): + MAX_CPU_TIME = 15 + # Exploit input from + # https://makenowjust-labs.github.io/recheck/playground/ + # for /(|\s.*?>))/ (old pattern to find the head element). + body = b"|\s.*?>))/ (old pattern to find the head element). - body = b" Date: Fri, 15 Dec 2023 11:49:22 +0100 Subject: [PATCH 10/18] Minor naming changes --- scrapy/utils/iterators.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 7574e377a..f239630f4 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -101,18 +101,18 @@ def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: if ":" not in element_name: return element_name, None, None reader: "SupportsReadClose[bytes]" = _StreamReader(data) - node_prefix, element_name = element_name.split(":", maxsplit=1) + input_prefix, element_name = element_name.split(":", maxsplit=1) ns_iterator = etree.iterparse( reader, encoding=reader.encoding, events=("start-ns",), **_ITERPARSE_KWARGS, ) - for event, (_prefix, _namespace) in ns_iterator: - if _prefix != node_prefix: + for event, (prefix, namespace) in ns_iterator: + if prefix != input_prefix: continue - return element_name, _prefix, _namespace - return f"{node_prefix}:{element_name}", None, None + return element_name, prefix, namespace + return f"{input_prefix}:{element_name}", None, None def xmliter_lxml( From a49c8762dd163b60cc73c4486a662471cfa7ac7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 12:14:53 +0100 Subject: [PATCH 11/18] Avoid calling iterparse twice --- scrapy/utils/iterators.py | 42 +++++++++++++++++---------------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index f239630f4..8610e9b77 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -12,7 +12,6 @@ from typing import ( List, Literal, Optional, - Tuple, Union, cast, overload, @@ -97,43 +96,38 @@ def xmliter( yield Selector(text=nodetext, type="xml") -def _resolve_xml_namespace(element_name: str, data: bytes) -> Tuple[str, str]: - if ":" not in element_name: - return element_name, None, None - reader: "SupportsReadClose[bytes]" = _StreamReader(data) - input_prefix, element_name = element_name.split(":", maxsplit=1) - ns_iterator = etree.iterparse( - reader, - encoding=reader.encoding, - events=("start-ns",), - **_ITERPARSE_KWARGS, - ) - for event, (prefix, namespace) in ns_iterator: - if prefix != input_prefix: - continue - return element_name, prefix, namespace - return f"{input_prefix}:{element_name}", None, None - - def xmliter_lxml( obj: Union[Response, str, bytes], nodename: str, namespace: Optional[str] = None, prefix: str = "x", ) -> Generator[Selector, Any, None]: - if not namespace: - nodename, prefix, namespace = _resolve_xml_namespace(nodename, obj) - reader: "SupportsReadClose[bytes]" = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( reader, - tag=tag, encoding=reader.encoding, + events=("end", "start-ns"), **_ITERPARSE_KWARGS, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) - for _, node in iterable: + needs_namespace_resolution = not namespace and ":" in nodename + if needs_namespace_resolution: + prefix, nodename = nodename.split(":", maxsplit=1) + for event, data in iterable: + if event == "start-ns": + if needs_namespace_resolution: + _prefix, _namespace = data + if _prefix != prefix: + continue + namespace = _namespace + needs_namespace_resolution = False + selxpath = f"//{prefix}:{nodename}" + tag = f"{{{namespace}}}{nodename}" + continue + node = data + if node.tag != tag: + continue nodetext = etree.tostring(node, encoding="unicode") node.clear() xs = Selector(text=nodetext, type="xml") From ce9d290eff8b5992023ffa5e833881b83a0669c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 12:23:26 +0100 Subject: [PATCH 12/18] Remove the lxml version check for huge_tree on xmliter_lxml iterparse supports the option since lxml 2.2.1, it was the HTML parser that only got it in 4.2 --- docs/news.rst | 2 +- scrapy/utils/iterators.py | 9 +-------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 525ddbf40..fab8b6f20 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -19,7 +19,7 @@ Scrapy 2.11.1 (unreleased) To minimize the impact of this change on existing code, :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating the node namespace with a prefix in the node name, and big files with - highly nested trees. + highly nested trees when using libxml2 2.7+. - Fixed regular expressions in the implementation of the :func:`~scrapy.utils.response.open_in_browser` function. diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index 8610e9b77..b6abe2e0c 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -19,7 +19,6 @@ from typing import ( from warnings import warn from lxml import etree -from packaging.version import Version from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Response, TextResponse @@ -31,12 +30,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -_LXML_VERSION = Version(etree.__version__) -_LXML_HUGE_TREE_VERSION = Version("4.2") -_ITERPARSE_KWARGS = {} -if _LXML_VERSION >= _LXML_HUGE_TREE_VERSION: - _ITERPARSE_KWARGS["huge_tree"] = True - def xmliter( obj: Union[Response, str, bytes], nodename: str @@ -108,7 +101,7 @@ def xmliter_lxml( reader, encoding=reader.encoding, events=("end", "start-ns"), - **_ITERPARSE_KWARGS, + huge_tree=True, ) selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename) needs_namespace_resolution = not namespace and ":" in nodename From bc138ef8e958f4bac5a4413d40566efc2b59acfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 12:24:04 +0100 Subject: [PATCH 13/18] Minor release notes fix --- docs/news.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/news.rst b/docs/news.rst index fab8b6f20..f346d1239 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -2912,7 +2912,7 @@ Scrapy 1.8.4 (unreleased) To minimize the impact of this change on existing code, :func:`~scrapy.utils.iterators.xmliter_lxml` now supports indicating the node namespace as a prefix in the node name, and big files with highly - nested trees when using lxml 4.2 or later. + nested trees when using libxml2 2.7+. Please, see the `cc65-xxvf-f7r9 security advisory`_ for more information. From c7c7a488b950806888691f58dda0b06478b98c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 13:18:23 +0100 Subject: [PATCH 14/18] Fix typing issues --- scrapy/utils/iterators.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py index b6abe2e0c..ab48e525f 100644 --- a/scrapy/utils/iterators.py +++ b/scrapy/utils/iterators.py @@ -95,10 +95,10 @@ def xmliter_lxml( namespace: Optional[str] = None, prefix: str = "x", ) -> Generator[Selector, Any, None]: - reader: "SupportsReadClose[bytes]" = _StreamReader(obj) + reader = _StreamReader(obj) tag = f"{{{namespace}}}{nodename}" if namespace else nodename iterable = etree.iterparse( - reader, + cast("SupportsReadClose[bytes]", reader), encoding=reader.encoding, events=("end", "start-ns"), huge_tree=True, @@ -109,6 +109,7 @@ def xmliter_lxml( prefix, nodename = nodename.split(":", maxsplit=1) for event, data in iterable: if event == "start-ns": + assert isinstance(data, tuple) if needs_namespace_resolution: _prefix, _namespace = data if _prefix != prefix: @@ -118,6 +119,7 @@ def xmliter_lxml( selxpath = f"//{prefix}:{nodename}" tag = f"{{{namespace}}}{nodename}" continue + assert isinstance(data, etree._Element) node = data if node.tag != tag: continue From 27781a85e738052e0441c81d773b3ec124194594 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 15 Dec 2023 13:52:12 +0100 Subject: [PATCH 15/18] Fix bad closing tags in XMLFeedSpider tests --- tests/test_spider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_spider.py b/tests/test_spider.py index 00da3d485..5c4007d87 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -149,10 +149,10 @@ class XMLFeedSpiderTest(SpiderTest): body = b""" - http://www.example.com/Special-Offers.html2009-08-16 + http://www.example.com/Special-Offers.html2009-08-16 - http://www.example.com/2009-08-16 + http://www.example.com/2009-08-16 """ response = XmlResponse(url="http://example.com/sitemap.xml", body=body) From c5dad41190551578c2973c34520952f26f75dc7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 2 Feb 2024 14:03:16 +0100 Subject: [PATCH 16/18] Speed up tests, remove comments without regexps --- .github/workflows/tests-ubuntu.yml | 3 -- scrapy/utils/response.py | 14 +++++++- tests/test_utils_response.py | 51 ++++++++++++++++++++++++++---- tox.ini | 6 ---- 4 files changed, 57 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 388ba9572..338c99584 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -50,9 +50,6 @@ jobs: - python-version: "3.12" env: TOXENV: botocore - - python-version: "3.12" - env: - TOXENV: slow steps: - uses: actions/checkout@v3 diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 4369e6439..fabfb1167 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -74,6 +74,18 @@ def response_httprepr(response: Response) -> bytes: return b"".join(values) +def _remove_html_comments(body): + start = body.find(b"", start + 1) + if end == -1: + return body[:start] + else: + body = body[:start] + body[end + 3 :] + start = body.find(b"|$)", b"", body) body = re.sub(rb"]*?>)", to_bytes(repl), body, count=1) ext = ".html" elif isinstance(response, TextResponse): diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 1dbe187bf..db3c31b89 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -8,9 +8,9 @@ import pytest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Response, TextResponse -from scrapy.settings.default_settings import DOWNLOAD_MAXSIZE from scrapy.utils.python import to_bytes from scrapy.utils.response import ( + _remove_html_comments, get_base_url, get_meta_refresh, open_in_browser, @@ -203,14 +203,13 @@ class ResponseUtilsTest(unittest.TestCase): r5, _openfunc=check_base_url ), "Inject unique base url with conditional comment" - @pytest.mark.slow def test_open_in_browser_redos_comment(self): - MAX_CPU_TIME = 30 + MAX_CPU_TIME = 0.001 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ # for // (old pattern to remove comments). - body = b"->" + body = b"->" response = HtmlResponse("https://example.com", body=body) @@ -221,14 +220,13 @@ class ResponseUtilsTest(unittest.TestCase): end_time = process_time() self.assertLess(end_time - start_time, MAX_CPU_TIME) - @pytest.mark.slow def test_open_in_browser_redos_head(self): - MAX_CPU_TIME = 15 + MAX_CPU_TIME = 0.001 # Exploit input from # https://makenowjust-labs.github.io/recheck/playground/ # for /(|\s.*?>))/ (old pattern to find the head element). - body = b"b", + b"ab", + ), + ( + b"ac", + b"ac", + ), + ( + b"acccd", + b"acd", + ), + ( + b"ad", + b"ad", + ), + ), +) +def test_remove_html_comments(input_body, output_body): + assert ( + _remove_html_comments(input_body) == output_body + ), f"{_remove_html_comments(input_body)=} == {output_body=}" diff --git a/tox.ini b/tox.ini index e87d6a175..381da9773 100644 --- a/tox.ini +++ b/tox.ini @@ -221,9 +221,3 @@ setenv = {[pinned]setenv} commands = pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} - - -[testenv:slow] -basepython = python3 -commands = - {[testenv]commands} -m 'slow' From 810aaa637da12a1f393291eb2b13aa0c8a163efb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 2 Feb 2024 14:04:28 +0100 Subject: [PATCH 17/18] Undo an unintended change --- .github/workflows/tests-ubuntu.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 338c99584..c883f958c 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -50,6 +50,7 @@ jobs: - python-version: "3.12" env: TOXENV: botocore + steps: - uses: actions/checkout@v3 From 5e5a92026e43023b80f7733844a2703c3f966009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 2 Feb 2024 14:06:45 +0100 Subject: [PATCH 18/18] Remove slow leftovers --- pytest.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/pytest.ini b/pytest.ini index 877fbcd1d..16983be5e 100644 --- a/pytest.ini +++ b/pytest.ini @@ -17,12 +17,10 @@ addopts = --ignore=docs/topics/stats.rst --ignore=docs/topics/telnetconsole.rst --ignore=docs/utils - -m 'not slow' markers = only_asyncio: marks tests as only enabled when --reactor=asyncio is passed only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed requires_uvloop: marks tests as only enabled when uvloop is known to be working - slow: marks tests as slow, not executed by default filterwarnings = ignore:scrapy.downloadermiddlewares.decompression is deprecated ignore:Module scrapy.utils.reqser is deprecated