From 13a014d2e6b5b9a3eca67a8f3d9b8e2c7b214f1d Mon Sep 17 00:00:00 2001 From: "Albert Eduardovich N." Date: Wed, 8 Apr 2026 15:25:11 +0300 Subject: [PATCH] Response now uses less memory (#7374) --- docs/news.rst | 20 ++++++++++++ scrapy/http/request/__init__.py | 8 +++-- scrapy/http/response/__init__.py | 52 +++++++++++++++++++++++++++----- scrapy/http/response/html.py | 2 +- scrapy/http/response/json.py | 2 +- scrapy/http/response/text.py | 17 ++++++++--- scrapy/http/response/xml.py | 2 +- tests/test_http_request.py | 4 +++ tests/test_http_response.py | 47 +++++++++++++++++++++++++++++ 9 files changed, 136 insertions(+), 18 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index e445b0498..4f9d8ab45 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -24,6 +24,26 @@ Backward-incompatible changes (:issue:`2183`, :issue:`6369`, :issue:`7182`) +- ``Request`` and ``Response`` objects: ``__slots__`` and setter changes: + + - :class:`scrapy.http.Request` and :class:`scrapy.http.Response` now + define ``__slots__``. Assigning arbitrary attributes to instances (for + example, ``response.foo = 1``) will raise ``AttributeError``. Store + per-request/response data in the request/response ``meta`` mapping + instead of attaching new attributes to the objects. + + - If you maintain custom ``Request`` or ``Response`` subclasses that + relied on dynamic instance attributes, either add ``'__dict__'`` to + your subclass ``__slots__`` to allow dynamic attributes, or migrate + per-instance state to ``meta`` or explicit documented attributes. + + - The setters for ``headers``, ``flags`` and ``cookies`` no longer coerce + falsy values into ``None``. For example, ``request.headers = {}`` now + stores an empty :class:`scrapy.http.headers.Headers` instance (not + ``None``), and ``request.flags = []`` remains an empty list instead of + being set to ``None``. Update code that relied on ``is None`` checks or + the previous coercion behaviour. + New features ~~~~~~~~~~~~ diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index f523c2bb9..00042e093 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -284,7 +284,7 @@ class Request(object_ref): @flags.setter def flags(self, value: list[str] | None) -> None: - self._flags = value or None + self._flags = value @property def cookies(self) -> CookiesT: @@ -294,7 +294,7 @@ class Request(object_ref): @cookies.setter def cookies(self, value: CookiesT | None) -> None: - self._cookies = value or None + self._cookies = value @property def headers(self) -> Headers: @@ -309,7 +309,9 @@ class Request(object_ref): if isinstance(value, Headers): self._headers = value else: - self._headers = Headers(value, encoding=self.encoding) if value else None + self._headers = ( + Headers(value, encoding=self.encoding) if value is not None else None + ) def __repr__(self) -> str: return f"<{self.method} {self.url}>" diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 7e23df491..a80ea3da8 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -38,17 +38,20 @@ class Response(object_ref): downloaded (by the Downloader) and fed to the Spiders for processing. """ - attributes: tuple[str, ...] = ( - "url", + __attrs_and_slots = ( "status", - "headers", - "body", - "flags", "request", "certificate", "ip_address", "protocol", ) + attributes: tuple[str, ...] = ( + "url", + "headers", + "body", + "flags", + *__attrs_and_slots, + ) """A tuple of :class:`str` objects containing the name of all public attributes of the class that are also keyword parameters of the ``__init__()`` method. @@ -56,6 +59,16 @@ class Response(object_ref): Currently used by :meth:`Response.replace`. """ + __slots__ = ( + "__weakref__", + "_url", + "_body", + "_headers", + "_flags", + *__attrs_and_slots, + ) + del __attrs_and_slots + def __init__( self, url: str, @@ -68,12 +81,12 @@ class Response(object_ref): ip_address: IPv4Address | IPv6Address | None = None, protocol: str | None = None, ): - self.headers: Headers = Headers(headers or {}) + self._headers: Headers | None = Headers(headers) if headers else None self.status: int = int(status) self._set_body(body) self._set_url(url) self.request: Request | None = request - self.flags: list[str] = [] if flags is None else list(flags) + self._flags: list[str] | None = list(flags) if flags else None self.certificate: Certificate | None = certificate self.ip_address: IPv4Address | IPv6Address | None = ip_address self.protocol: str | None = protocol @@ -125,6 +138,31 @@ class Response(object_ref): else: self._body = body + @property + def headers(self) -> Headers: + if self._headers is None: + self._headers = Headers() + return self._headers + + @headers.setter + def headers( + self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None + ) -> None: + if isinstance(value, Headers): + self._headers = value + else: + self._headers = Headers(value) if value is not None else None + + @property + def flags(self) -> list[str]: + if self._flags is None: + self._flags = [] + return self._flags + + @flags.setter + def flags(self, value: list[str] | None) -> None: + self._flags = value + def __repr__(self) -> str: return f"<{self.status} {self.url}>" diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index 7eed052c2..70c08c11d 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -9,4 +9,4 @@ from scrapy.http.response.text import TextResponse class HtmlResponse(TextResponse): - pass + __slots__ = () diff --git a/scrapy/http/response/json.py b/scrapy/http/response/json.py index 219691094..0428dde6e 100644 --- a/scrapy/http/response/json.py +++ b/scrapy/http/response/json.py @@ -9,4 +9,4 @@ from scrapy.http.response.text import TextResponse class JsonResponse(TextResponse): - pass + __slots__ = () diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 077b86a33..13853f64d 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -41,15 +41,22 @@ _NONE = object() class TextResponse(Response): _DEFAULT_ENCODING = "ascii" - _cached_decoded_json = _NONE attributes: tuple[str, ...] = (*Response.attributes, "encoding") + __slots__ = ( + "_cached_benc", + "_cached_decoded_json", + "_cached_selector", + "_cached_ubody", + "_encoding", + ) def __init__(self, *args: Any, **kwargs: Any): self._encoding: str | None = kwargs.pop("encoding", None) self._cached_benc: str | None = None self._cached_ubody: str | None = None self._cached_selector: Selector | None = None + self._cached_decoded_json: object = _NONE super().__init__(*args, **kwargs) def _set_body(self, body: str | bytes | None) -> None: @@ -100,7 +107,7 @@ class TextResponse(Response): @memoizemethod_noargs def _headers_encoding(self) -> str | None: - content_type = cast("bytes", self.headers.get(b"Content-Type", b"")) + content_type = self.headers.get(b"Content-Type") or b"" return http_content_type_encoding(to_unicode(content_type, encoding="latin-1")) def _body_inferred_encoding(self) -> str: @@ -138,10 +145,10 @@ class TextResponse(Response): @property def selector(self) -> Selector: - # circular import - from scrapy.selector import Selector # noqa: PLC0415 - if self._cached_selector is None: + # circular import + from scrapy.selector import Selector # noqa: PLC0415 + self._cached_selector = Selector(self) return self._cached_selector diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py index abf474a2f..6d9c4cb73 100644 --- a/scrapy/http/response/xml.py +++ b/scrapy/http/response/xml.py @@ -9,4 +9,4 @@ from scrapy.http.response.text import TextResponse class XmlResponse(TextResponse): - pass + __slots__ = () diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 81f8fa3c1..4c77bb1ec 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -349,6 +349,7 @@ class TestRequest: assert request._flags == [] original_flags = request.flags request.flags = None + assert request._flags is None assert request.flags == [] assert request.flags is not original_flags @@ -358,6 +359,7 @@ class TestRequest: assert request._cookies == {} original_cookies = request.cookies request.cookies = None + assert request._cookies is None assert request.cookies == {} assert request.cookies is not original_cookies @@ -373,7 +375,9 @@ class TestRequest: assert isinstance(request._headers, Headers) original_headers = request.headers request.headers = None + assert request._headers is None assert request.headers == {} + assert request._headers == {} assert request.headers is not original_headers def test_no_callback(self): diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 1becf1542..09c95dc29 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -160,6 +160,53 @@ class TestResponse: with pytest.raises(AttributeError): r.body = "xxx" + def test_setter_mutable_lazy_loading(self): + """Mutable attributes are set internally to None only until they are + read, then they always return the same falsy instance of the + corresponding mutable structure. + + Setting them to None causes the next read to return a different object. + """ + + response = self.response_class("http://example.com") + + response.request = Request("http://example.com") + + assert response._flags is None + assert response.flags == [] + assert response.flags is response.flags + assert response._flags == [] + original_flags = response.flags + response.flags = None + assert response._flags is None + assert response.flags == [] + assert response.flags is not original_flags + + assert response._headers is None + assert response.headers == {} + assert response.headers is response.headers + assert isinstance(response.headers, Headers) + assert isinstance(response._headers, Headers) + original_headers = response.headers + response.headers = None + assert response._headers is None + assert response.headers == {} + assert response._headers == {} + assert response.headers is not original_headers + + def test_setters(self): + response = self.response_class("http://example.com") + + response.flags = ["f1"] + assert response.flags == ["f1"] + + headers = Headers({b"X-Test": b"1"}) + response.headers = headers + assert response._headers is headers + response.headers = {b"A": b"b"} + assert isinstance(response.headers, Headers) + assert response._headers[b"A"] == b"b" + def test_urljoin(self): """Test urljoin shortcut (only for existence, since behavior equals urljoin)""" joined = self.response_class("http://www.example.com").urljoin("/test")