Reducing memory footprint of `Request` object: __slots__ and lazy evaluation (#7036)

* reducing memory foorprint of `Request` object

* restore flow

* Test setters

---------

Co-authored-by: Adrian Chaves <adrian@zyte.com>

scrapy/http/request/__init__.py

@@ -86,20 +86,17 @@ class Request(object_ref):
     executed by the Downloader, thus generating a :class:`~scrapy.http.Response`.
     """
 
+    __attrs_and_slots = ("callback", "dont_filter", "errback", "method", "priority")
     attributes: tuple[str, ...] = (
         "url",
-        "callback",
-        "method",
         "headers",
         "body",
         "cookies",
         "meta",
         "encoding",
-        "priority",
-        "dont_filter",
-        "errback",
         "flags",
         "cb_kwargs",
+        *__attrs_and_slots,
     )
     """A tuple of :class:`str` objects containing the name of all public
     attributes of the class that are also keyword parameters of the
@@ -109,6 +106,20 @@ class Request(object_ref):
     :func:`~scrapy.utils.request.request_from_dict`.
     """
 
+    __slots__ = (
+        "__weakref__",
+        "_body",
+        "_cb_kwargs",
+        "_cookies",
+        "_encoding",
+        "_flags",
+        "_headers",
+        "_meta",
+        "_url",
+        *__attrs_and_slots,
+    )
+    del __attrs_and_slots
+
     def __init__(
         self,
         url: str,
@@ -189,8 +200,10 @@ class Request(object_ref):
         #: .. seealso:: :ref:`topics-request-response-ref-errbacks`
         self.errback: Callable[[Failure], Any] | None = errback
 
-        self.cookies: CookiesT = cookies or {}
-        self.headers: Headers = Headers(headers or {}, encoding=encoding)
+        self._cookies: CookiesT | None = cookies if cookies else None
+        self._headers: Headers | None = (
+            Headers(headers, encoding=encoding) if headers else None
+        )
 
         #: Whether this request may be filtered out by :ref:`components
         #: <topics-components>` that support filtering out requests (``False``,
@@ -207,7 +220,7 @@ class Request(object_ref):
 
         self._meta: dict[str, Any] | None = dict(meta) if meta else None
         self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else None
-        self.flags: list[str] = [] if flags is None else list(flags)
+        self._flags: list[str] | None = list(flags) if flags else None
 
     @property
     def cb_kwargs(self) -> dict[str, Any]:
@@ -215,12 +228,20 @@ class Request(object_ref):
             self._cb_kwargs = {}
         return self._cb_kwargs
 
+    @cb_kwargs.setter
+    def cb_kwargs(self, value: dict[str, Any] | None) -> None:
+        self._cb_kwargs = value if value else None
+
     @property
     def meta(self) -> dict[str, Any]:
         if self._meta is None:
             self._meta = {}
         return self._meta
 
+    @meta.setter
+    def meta(self, value: dict[str, Any] | None) -> None:
+        self._meta = value if value else None
+
     @property
     def url(self) -> str:
         return self._url
@@ -243,12 +264,47 @@ class Request(object_ref):
         return self._body
 
     def _set_body(self, body: str | bytes | None) -> None:
-        self._body = b"" if body is None else to_bytes(body, self.encoding)
+        self._body = b"" if not body else to_bytes(body, self.encoding)
 
     @property
     def encoding(self) -> str:
         return self._encoding
 
+    @property
+    def flags(self) -> list[str]:
+        if self._flags is None:
+            self._flags = []
+        return self._flags
+
+    @flags.setter
+    def flags(self, value: list[str] | None) -> None:
+        self._flags = value if value else None
+
+    @property
+    def cookies(self) -> CookiesT:
+        if self._cookies is None:
+            self._cookies = {}
+        return self._cookies
+
+    @cookies.setter
+    def cookies(self, value: CookiesT | None) -> None:
+        self._cookies = value if value else None
+
+    @property
+    def headers(self) -> Headers:
+        if self._headers is None:
+            self._headers = Headers(encoding=self.encoding)
+        return self._headers
+
+    @headers.setter
+    def headers(
+        self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
+    ) -> None:
+        if isinstance(value, Headers):
+            self._headers = value
+        else:
+            self._headers = Headers(value, encoding=self.encoding) if value else None
+
     def __repr__(self) -> str:
         return f"<{self.method} {self.url}>"
 

scrapy/http/request/form.py

@@ -37,6 +37,8 @@ FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None
 
 
 class FormRequest(Request):
+    __slots__ = ()
+
     valid_form_methods = ["GET", "POST"]
 
     def __init__(

scrapy/http/request/json_request.py

@@ -20,6 +20,8 @@ if TYPE_CHECKING:
 
 
 class JsonRequest(Request):
+    __slots__ = ("_dumps_kwargs",)
+
     attributes: tuple[str, ...] = (*Request.attributes, "dumps_kwargs")
 
     def __init__(

scrapy/http/request/rpc.py

@@ -21,6 +21,8 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
 
 
 class XmlRpcRequest(Request):
+    __slots__ = ()
+
     def __init__(self, *args: Any, encoding: str | None = None, **kwargs: Any):
         if "body" not in kwargs and "params" in kwargs:
             kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs}

tests/test_http_request.py

@@ -329,6 +329,89 @@ class TestRequest:
                 errback="a_function",
             )
 
+    def test_setters(self):
+        request = self.request_class("http://example.com")
+
+        request.cb_kwargs = {"a": 1}
+        assert request.cb_kwargs == {"a": 1}
+
+        request.meta = {"k": "v"}
+        assert request.meta == {"k": "v"}
+
+        request.flags = ["f1"]
+        assert request.flags == ["f1"]
+
+        request.cookies = {"sid": "1"}
+        assert request.cookies == {"sid": "1"}
+
+        headers = Headers({b"X-Test": b"1"})
+        request.headers = headers
+        assert request._headers is headers
+        request.headers = {b"A": b"b"}
+        assert isinstance(request.headers, Headers)
+        assert request._headers[b"A"] == b"b"
+
+    def test_setter_mutable_lazy_loading(self):
+        """Mutable attributes are set internally to None only until they are
+        read, then they always return the same falsy instance of the
+        corresponding mutable structure.
+
+        Setting them to None causes the next read to return a different object.
+        """
+
+        request = self.request_class("http://example.com")
+
+        assert request._cb_kwargs is None
+        assert request.cb_kwargs == {}
+        assert request.cb_kwargs is request.cb_kwargs
+        assert request._cb_kwargs == {}
+        original_cb_kwargs = request.cb_kwargs
+        request.cb_kwargs = None
+        assert request.cb_kwargs == {}
+        assert request.cb_kwargs is not original_cb_kwargs
+
+        assert request._meta is None
+        assert request.meta == {}
+        assert request.meta is request.meta
+        assert request._meta == {}
+        original_meta = request.meta
+        request.meta = None
+        assert request.meta == {}
+        assert request.meta is not original_meta
+
+        assert request._flags is None
+        assert request.flags == []
+        assert request.flags is request.flags
+        assert request._flags == []
+        original_flags = request.flags
+        request.flags = None
+        assert request.flags == []
+        assert request.flags is not original_flags
+
+        assert request._cookies is None
+        assert request.cookies == {}
+        assert request.cookies is request.cookies
+        assert request._cookies == {}
+        original_cookies = request.cookies
+        request.cookies = None
+        assert request.cookies == {}
+        assert request.cookies is not original_cookies
+
+        if self.default_headers:
+            assert request._headers == self.default_headers
+            assert request._headers is not self.default_headers
+            assert request.headers == self.default_headers
+        else:
+            assert request._headers is None
+            assert request.headers == {}
+        assert request.headers is request.headers
+        assert isinstance(request.headers, Headers)
+        assert isinstance(request._headers, Headers)
+        original_headers = request.headers
+        request.headers = None
+        assert request.headers == {}
+        assert request.headers is not original_headers
+
     def test_no_callback(self):
         with pytest.raises(RuntimeError):
             NO_CALLBACK()