Response now uses less memory (#7374)

docs/news.rst

@@ -24,6 +24,26 @@ Backward-incompatible changes
 
     (:issue:`2183`, :issue:`6369`, :issue:`7182`)
 
+-   ``Request`` and ``Response`` objects: ``__slots__`` and setter changes:
+
+    -   :class:`scrapy.http.Request` and :class:`scrapy.http.Response` now
+        define ``__slots__``. Assigning arbitrary attributes to instances (for
+        example, ``response.foo = 1``) will raise ``AttributeError``. Store
+        per-request/response data in the request/response ``meta`` mapping
+        instead of attaching new attributes to the objects.
+
+    -   If you maintain custom ``Request`` or ``Response`` subclasses that
+        relied on dynamic instance attributes, either add ``'__dict__'`` to
+        your subclass ``__slots__`` to allow dynamic attributes, or migrate
+        per-instance state to ``meta`` or explicit documented attributes.
+
+    -   The setters for ``headers``, ``flags`` and ``cookies`` no longer coerce
+        falsy values into ``None``. For example, ``request.headers = {}`` now
+        stores an empty :class:`scrapy.http.headers.Headers` instance (not
+        ``None``), and ``request.flags = []`` remains an empty list instead of
+        being set to ``None``. Update code that relied on ``is None`` checks or
+        the previous coercion behaviour.
+
 New features
 ~~~~~~~~~~~~
 

scrapy/http/request/__init__.py

@@ -284,7 +284,7 @@ class Request(object_ref):
 
     @flags.setter
     def flags(self, value: list[str] | None) -> None:
-        self._flags = value or None
+        self._flags = value
 
     @property
     def cookies(self) -> CookiesT:
@@ -294,7 +294,7 @@ class Request(object_ref):
 
     @cookies.setter
     def cookies(self, value: CookiesT | None) -> None:
-        self._cookies = value or None
+        self._cookies = value
 
     @property
     def headers(self) -> Headers:
@@ -309,7 +309,9 @@ class Request(object_ref):
         if isinstance(value, Headers):
             self._headers = value
         else:
-            self._headers = Headers(value, encoding=self.encoding) if value else None
+            self._headers = (
+                Headers(value, encoding=self.encoding) if value is not None else None
+            )
 
     def __repr__(self) -> str:
         return f"<{self.method} {self.url}>"

scrapy/http/response/__init__.py

@@ -38,17 +38,20 @@ class Response(object_ref):
     downloaded (by the Downloader) and fed to the Spiders for processing.
     """
 
-    attributes: tuple[str, ...] = (
-        "url",
+    __attrs_and_slots = (
         "status",
-        "headers",
-        "body",
-        "flags",
         "request",
         "certificate",
         "ip_address",
         "protocol",
     )
+    attributes: tuple[str, ...] = (
+        "url",
+        "headers",
+        "body",
+        "flags",
+        *__attrs_and_slots,
+    )
     """A tuple of :class:`str` objects containing the name of all public
     attributes of the class that are also keyword parameters of the
     ``__init__()`` method.
@@ -56,6 +59,16 @@ class Response(object_ref):
     Currently used by :meth:`Response.replace`.
     """
 
+    __slots__ = (
+        "__weakref__",
+        "_url",
+        "_body",
+        "_headers",
+        "_flags",
+        *__attrs_and_slots,
+    )
+    del __attrs_and_slots
+
     def __init__(
         self,
         url: str,
@@ -68,12 +81,12 @@ class Response(object_ref):
         ip_address: IPv4Address | IPv6Address | None = None,
         protocol: str | None = None,
     ):
-        self.headers: Headers = Headers(headers or {})
+        self._headers: Headers | None = Headers(headers) if headers else None
         self.status: int = int(status)
         self._set_body(body)
         self._set_url(url)
         self.request: Request | None = request
-        self.flags: list[str] = [] if flags is None else list(flags)
+        self._flags: list[str] | None = list(flags) if flags else None
         self.certificate: Certificate | None = certificate
         self.ip_address: IPv4Address | IPv6Address | None = ip_address
         self.protocol: str | None = protocol
@@ -125,6 +138,31 @@ class Response(object_ref):
         else:
             self._body = body
 
+    @property
+    def headers(self) -> Headers:
+        if self._headers is None:
+            self._headers = Headers()
+        return self._headers
+
+    @headers.setter
+    def headers(
+        self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
+    ) -> None:
+        if isinstance(value, Headers):
+            self._headers = value
+        else:
+            self._headers = Headers(value) if value is not None else None
+
+    @property
+    def flags(self) -> list[str]:
+        if self._flags is None:
+            self._flags = []
+        return self._flags
+
+    @flags.setter
+    def flags(self, value: list[str] | None) -> None:
+        self._flags = value
+
     def __repr__(self) -> str:
         return f"<{self.status} {self.url}>"
 

scrapy/http/response/html.py

@@ -9,4 +9,4 @@ from scrapy.http.response.text import TextResponse
 
 
 class HtmlResponse(TextResponse):
-    pass
+    __slots__ = ()

scrapy/http/response/json.py

@@ -9,4 +9,4 @@ from scrapy.http.response.text import TextResponse
 
 
 class JsonResponse(TextResponse):
-    pass
+    __slots__ = ()

scrapy/http/response/text.py

@@ -41,15 +41,22 @@ _NONE = object()
 
 class TextResponse(Response):
     _DEFAULT_ENCODING = "ascii"
-    _cached_decoded_json = _NONE
 
     attributes: tuple[str, ...] = (*Response.attributes, "encoding")
+    __slots__ = (
+        "_cached_benc",
+        "_cached_decoded_json",
+        "_cached_selector",
+        "_cached_ubody",
+        "_encoding",
+    )
 
     def __init__(self, *args: Any, **kwargs: Any):
         self._encoding: str | None = kwargs.pop("encoding", None)
         self._cached_benc: str | None = None
         self._cached_ubody: str | None = None
         self._cached_selector: Selector | None = None
+        self._cached_decoded_json: object = _NONE
         super().__init__(*args, **kwargs)
 
     def _set_body(self, body: str | bytes | None) -> None:
@@ -100,7 +107,7 @@ class TextResponse(Response):
 
     @memoizemethod_noargs
     def _headers_encoding(self) -> str | None:
-        content_type = cast("bytes", self.headers.get(b"Content-Type", b""))
+        content_type = self.headers.get(b"Content-Type") or b""
         return http_content_type_encoding(to_unicode(content_type, encoding="latin-1"))
 
     def _body_inferred_encoding(self) -> str:
@@ -138,10 +145,10 @@ class TextResponse(Response):
 
     @property
     def selector(self) -> Selector:
-        # circular import
-        from scrapy.selector import Selector  # noqa: PLC0415
-
         if self._cached_selector is None:
+            # circular import
+            from scrapy.selector import Selector  # noqa: PLC0415
+
             self._cached_selector = Selector(self)
         return self._cached_selector
 

scrapy/http/response/xml.py

@@ -9,4 +9,4 @@ from scrapy.http.response.text import TextResponse
 
 
 class XmlResponse(TextResponse):
-    pass
+    __slots__ = ()

tests/test_http_request.py

@@ -349,6 +349,7 @@ class TestRequest:
         assert request._flags == []
         original_flags = request.flags
         request.flags = None
+        assert request._flags is None
         assert request.flags == []
         assert request.flags is not original_flags
 
@@ -358,6 +359,7 @@ class TestRequest:
         assert request._cookies == {}
         original_cookies = request.cookies
         request.cookies = None
+        assert request._cookies is None
         assert request.cookies == {}
         assert request.cookies is not original_cookies
 
@@ -373,7 +375,9 @@ class TestRequest:
         assert isinstance(request._headers, Headers)
         original_headers = request.headers
         request.headers = None
+        assert request._headers is None
         assert request.headers == {}
+        assert request._headers == {}
         assert request.headers is not original_headers
 
     def test_no_callback(self):

tests/test_http_response.py

@@ -160,6 +160,53 @@ class TestResponse:
         with pytest.raises(AttributeError):
             r.body = "xxx"
 
+    def test_setter_mutable_lazy_loading(self):
+        """Mutable attributes are set internally to None only until they are
+        read, then they always return the same falsy instance of the
+        corresponding mutable structure.
+
+        Setting them to None causes the next read to return a different object.
+        """
+
+        response = self.response_class("http://example.com")
+
+        response.request = Request("http://example.com")
+
+        assert response._flags is None
+        assert response.flags == []
+        assert response.flags is response.flags
+        assert response._flags == []
+        original_flags = response.flags
+        response.flags = None
+        assert response._flags is None
+        assert response.flags == []
+        assert response.flags is not original_flags
+
+        assert response._headers is None
+        assert response.headers == {}
+        assert response.headers is response.headers
+        assert isinstance(response.headers, Headers)
+        assert isinstance(response._headers, Headers)
+        original_headers = response.headers
+        response.headers = None
+        assert response._headers is None
+        assert response.headers == {}
+        assert response._headers == {}
+        assert response.headers is not original_headers
+
+    def test_setters(self):
+        response = self.response_class("http://example.com")
+
+        response.flags = ["f1"]
+        assert response.flags == ["f1"]
+
+        headers = Headers({b"X-Test": b"1"})
+        response.headers = headers
+        assert response._headers is headers
+        response.headers = {b"A": b"b"}
+        assert isinstance(response.headers, Headers)
+        assert response._headers[b"A"] == b"b"
+
     def test_urljoin(self):
         """Test urljoin shortcut (only for existence, since behavior equals urljoin)"""
         joined = self.response_class("http://www.example.com").urljoin("/test")