diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index f0d20843f..d306f1f6e 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -86,20 +86,17 @@ class Request(object_ref): executed by the Downloader, thus generating a :class:`~scrapy.http.Response`. """ + __attrs_and_slots = ("callback", "dont_filter", "errback", "method", "priority") attributes: tuple[str, ...] = ( "url", - "callback", - "method", "headers", "body", "cookies", "meta", "encoding", - "priority", - "dont_filter", - "errback", "flags", "cb_kwargs", + *__attrs_and_slots, ) """A tuple of :class:`str` objects containing the name of all public attributes of the class that are also keyword parameters of the @@ -109,6 +106,20 @@ class Request(object_ref): :func:`~scrapy.utils.request.request_from_dict`. """ + __slots__ = ( + "__weakref__", + "_body", + "_cb_kwargs", + "_cookies", + "_encoding", + "_flags", + "_headers", + "_meta", + "_url", + *__attrs_and_slots, + ) + del __attrs_and_slots + def __init__( self, url: str, @@ -189,8 +200,10 @@ class Request(object_ref): #: .. seealso:: :ref:`topics-request-response-ref-errbacks` self.errback: Callable[[Failure], Any] | None = errback - self.cookies: CookiesT = cookies or {} - self.headers: Headers = Headers(headers or {}, encoding=encoding) + self._cookies: CookiesT | None = cookies if cookies else None + self._headers: Headers | None = ( + Headers(headers, encoding=encoding) if headers else None + ) #: Whether this request may be filtered out by :ref:`components #: ` that support filtering out requests (``False``, @@ -207,7 +220,7 @@ class Request(object_ref): self._meta: dict[str, Any] | None = dict(meta) if meta else None self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else None - self.flags: list[str] = [] if flags is None else list(flags) + self._flags: list[str] | None = list(flags) if flags else None @property def cb_kwargs(self) -> dict[str, Any]: @@ -215,12 +228,20 @@ class Request(object_ref): self._cb_kwargs = {} return self._cb_kwargs + @cb_kwargs.setter + def cb_kwargs(self, value: dict[str, Any] | None) -> None: + self._cb_kwargs = value if value else None + @property def meta(self) -> dict[str, Any]: if self._meta is None: self._meta = {} return self._meta + @meta.setter + def meta(self, value: dict[str, Any] | None) -> None: + self._meta = value if value else None + @property def url(self) -> str: return self._url @@ -243,12 +264,47 @@ class Request(object_ref): return self._body def _set_body(self, body: str | bytes | None) -> None: - self._body = b"" if body is None else to_bytes(body, self.encoding) + self._body = b"" if not body else to_bytes(body, self.encoding) @property def encoding(self) -> str: return self._encoding + @property + def flags(self) -> list[str]: + if self._flags is None: + self._flags = [] + return self._flags + + @flags.setter + def flags(self, value: list[str] | None) -> None: + self._flags = value if value else None + + @property + def cookies(self) -> CookiesT: + if self._cookies is None: + self._cookies = {} + return self._cookies + + @cookies.setter + def cookies(self, value: CookiesT | None) -> None: + self._cookies = value if value else None + + @property + def headers(self) -> Headers: + if self._headers is None: + self._headers = Headers(encoding=self.encoding) + return self._headers + + @headers.setter + def headers( + self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None + ) -> None: + if isinstance(value, Headers): + self._headers = value + else: + self._headers = Headers(value, encoding=self.encoding) if value else None + def __repr__(self) -> str: return f"<{self.method} {self.url}>" diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 7b2e480c4..7cb8bfd5d 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -37,6 +37,8 @@ FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None class FormRequest(Request): + __slots__ = () + valid_form_methods = ["GET", "POST"] def __init__( diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index e26cbe05b..1776bdca8 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -20,6 +20,8 @@ if TYPE_CHECKING: class JsonRequest(Request): + __slots__ = ("_dumps_kwargs",) + attributes: tuple[str, ...] = (*Request.attributes, "dumps_kwargs") def __init__( diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 01fe740a8..c68c8aabd 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -21,6 +21,8 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): + __slots__ = () + def __init__(self, *args: Any, encoding: str | None = None, **kwargs: Any): if "body" not in kwargs and "params" in kwargs: kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs} diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 22f44bfd2..93cbcd1eb 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -329,6 +329,89 @@ class TestRequest: errback="a_function", ) + def test_setters(self): + request = self.request_class("http://example.com") + + request.cb_kwargs = {"a": 1} + assert request.cb_kwargs == {"a": 1} + + request.meta = {"k": "v"} + assert request.meta == {"k": "v"} + + request.flags = ["f1"] + assert request.flags == ["f1"] + + request.cookies = {"sid": "1"} + assert request.cookies == {"sid": "1"} + + headers = Headers({b"X-Test": b"1"}) + request.headers = headers + assert request._headers is headers + request.headers = {b"A": b"b"} + assert isinstance(request.headers, Headers) + assert request._headers[b"A"] == b"b" + + def test_setter_mutable_lazy_loading(self): + """Mutable attributes are set internally to None only until they are + read, then they always return the same falsy instance of the + corresponding mutable structure. + + Setting them to None causes the next read to return a different object. + """ + + request = self.request_class("http://example.com") + + assert request._cb_kwargs is None + assert request.cb_kwargs == {} + assert request.cb_kwargs is request.cb_kwargs + assert request._cb_kwargs == {} + original_cb_kwargs = request.cb_kwargs + request.cb_kwargs = None + assert request.cb_kwargs == {} + assert request.cb_kwargs is not original_cb_kwargs + + assert request._meta is None + assert request.meta == {} + assert request.meta is request.meta + assert request._meta == {} + original_meta = request.meta + request.meta = None + assert request.meta == {} + assert request.meta is not original_meta + + assert request._flags is None + assert request.flags == [] + assert request.flags is request.flags + assert request._flags == [] + original_flags = request.flags + request.flags = None + assert request.flags == [] + assert request.flags is not original_flags + + assert request._cookies is None + assert request.cookies == {} + assert request.cookies is request.cookies + assert request._cookies == {} + original_cookies = request.cookies + request.cookies = None + assert request.cookies == {} + assert request.cookies is not original_cookies + + if self.default_headers: + assert request._headers == self.default_headers + assert request._headers is not self.default_headers + assert request.headers == self.default_headers + else: + assert request._headers is None + assert request.headers == {} + assert request.headers is request.headers + assert isinstance(request.headers, Headers) + assert isinstance(request._headers, Headers) + original_headers = request.headers + request.headers = None + assert request.headers == {} + assert request.headers is not original_headers + def test_no_callback(self): with pytest.raises(RuntimeError): NO_CALLBACK()