diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index dec9904d2..4893c6174 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -466,6 +466,9 @@ Filesystem storage backend (default) * ``response_headers`` - the response headers (in raw HTTP format) + * ``response_data`` - the remaining data of the response, as returned by + :meth:`Response.to_dict() `, pickled + * ``meta`` - some metadata of this cache resource in Python ``repr()`` format (grep-friendly format) @@ -499,6 +502,11 @@ Writing your own storage backend You can implement a cache storage backend by creating a Python class that defines the methods described below. +To store a response, use :meth:`Response.to_dict() +`, and to read it back, use +:func:`~scrapy.utils.response.response_from_dict`. That way responses of any +class, including those of third-party plugins, are cached and restored intact. + .. module:: scrapy.extensions.httpcache .. class:: CacheStorage diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 1d97e39b6..3c2b2a75b 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -1267,6 +1267,10 @@ Response objects .. automethod:: Response.follow_all + .. automethod:: Response.to_dict + + .. automethod:: Response.from_dict + .. _topics-request-response-ref-response-subclasses: @@ -1420,3 +1424,9 @@ JsonResponse objects that is used when the response has a `JSON MIME type `_ in its `Content-Type` header. + + +Other functions related to responses +==================================== + +.. autofunction:: scrapy.utils.response.response_from_dict diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index d008d0f67..f7b1c73d4 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -12,11 +12,11 @@ from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict -from scrapy.http import Headers, Response -from scrapy.responsetypes import responsetypes +from scrapy.http import Response from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.project import data_path from scrapy.utils.python import to_bytes, to_unicode +from scrapy.utils.response import response_from_dict if TYPE_CHECKING: import os @@ -272,24 +272,13 @@ class DbmCacheStorage: data = self._read_data(spider, request) if data is None: return None # not cached - url = data["url"] - status = data["status"] - headers = Headers(data["headers"]) - body = data["body"] - respcls = responsetypes.from_args(headers=headers, url=url, body=body) - return respcls(url=url, headers=headers, status=status, body=body) + return response_from_dict(data) def store_response( self, spider: Spider, request: Request, response: Response ) -> None: key = self._fingerprinter.fingerprint(request).hex() - data = { - "status": response.status, - "url": response.url, - "headers": dict(response.headers), - "body": response.body, - } - self.db[f"{key}_data"] = pickle.dumps(data, protocol=4) + self.db[f"{key}_data"] = pickle.dumps(response.to_dict(), protocol=4) self.db[f"{key}_time"] = str(time()) def _read_data(self, spider: Spider, request: Request) -> dict[str, Any] | None: @@ -340,11 +329,17 @@ class FilesystemCacheStorage: body = f.read() with self._open(rpath / "response_headers", "rb") as f: rawheaders = f.read() - url = metadata["response_url"] - status = metadata["status"] - headers = Headers(headers_raw_to_dict(rawheaders)) - respcls = responsetypes.from_args(headers=headers, url=url, body=body) - return respcls(url=url, headers=headers, status=status, body=body) + data = { + "url": metadata["response_url"], + "status": metadata["status"], + "headers": headers_raw_to_dict(rawheaders), + "body": body, + } + datapath = rpath / "response_data" + if datapath.exists(): + with self._open(datapath, "rb") as f: + data.update(pickle.load(f)) # noqa: S301 + return response_from_dict(data) def store_response( self, spider: Spider, request: Request, response: Response @@ -368,6 +363,13 @@ class FilesystemCacheStorage: f.write(headers_dict_to_raw(response.headers)) with self._open(rpath / "response_body", "wb") as f: f.write(response.body) + data = { + key: value + for key, value in response.to_dict().items() + if key not in {"url", "status", "headers", "body"} + } + with self._open(rpath / "response_data", "wb") as f: + pickle.dump(data, f, protocol=4) with self._open(rpath / "request_headers", "wb") as f: f.write(headers_dict_to_raw(request.headers)) with self._open(rpath / "request_body", "wb") as f: diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f91cb2095..2f148ff84 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -194,6 +194,37 @@ class Response(object_ref): cls = self.__class__ return cls(*args, **kwargs) + def to_dict(self) -> dict[str, Any]: + """Return a dictionary containing the Response's data. + + .. versionadded:: VERSION + + Use :func:`~scrapy.utils.response.response_from_dict` to convert back + into a :class:`~scrapy.http.Response` object. + + :attr:`request` and :attr:`certificate` are left out, as they are tied + to a single crawl. Everything else in :attr:`attributes` is included, + so subclasses only need to override this method, and :meth:`from_dict`, + if some of their attributes cannot be stored as is. + """ + d: dict[str, Any] = {"headers": dict(self.headers)} + for attr in self.attributes: + if attr in {"request", "certificate"}: + continue + d.setdefault(attr, getattr(self, attr)) + if type(self) is not Response: # pylint: disable=unidiomatic-typecheck + d["_class"] = self.__module__ + "." + self.__class__.__name__ + return d + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> Self: + """Return a response built from the *d* dict, as returned by + :meth:`to_dict`. + + .. versionadded:: VERSION + """ + return cls(**{key: value for key, value in d.items() if key != "_class"}) + def urljoin(self, url: str) -> str: """Join this Response's url with a possible relative url to form an absolute interpretation of the latter.""" diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index b3622c159..f0579beb7 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -15,6 +15,8 @@ from weakref import WeakKeyDictionary from twisted.web import http from w3lib import html +from scrapy.http.headers import Headers +from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode if TYPE_CHECKING: @@ -53,6 +55,30 @@ def get_meta_refresh( return _metaref_cache[response] +def response_from_dict(d: dict[str, Any]) -> Response: + """Return a response built from the *d* dict, as returned by + :meth:`Response.to_dict() `. + + .. versionadded:: VERSION + + If *d* does not indicate a response class, e.g. because it comes from a + plain :class:`~scrapy.http.Response` object or predates :meth:`~scrapy.http.Response.to_dict`, + the class is guessed with :attr:`~scrapy.responsetypes.responsetypes`. + """ + # Imported here to avoid a circular import. + from scrapy.responsetypes import responsetypes # noqa: PLC0415 + + d = {**d, "headers": Headers(d.get("headers") or {})} + response_cls: type[Response] = ( + load_object(d["_class"]) + if "_class" in d + else responsetypes.from_args( + headers=d["headers"], url=d["url"], body=d.get("body") + ) + ) + return response_cls.from_dict(d) + + def response_status_message(status: bytes | float | str) -> str: """Return status code plus status text descriptive message""" status_int = int(status) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 6eabe91ad..7eb3f14dc 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -2,6 +2,7 @@ from __future__ import annotations import email.utils import logging +import pickle import shutil import tempfile import time @@ -26,6 +27,14 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler +class CustomResponse(Response): + attributes: tuple[str, ...] = (*Response.attributes, "custom") + + def __init__(self, *args: Any, custom: str | None = None, **kwargs: Any): + self.custom = custom + super().__init__(*args, **kwargs) + + class AlwaysStalePolicy(DummyPolicy): """:class:`~scrapy.extensions.httpcache.DummyPolicy` that always revalidates cached responses.""" @@ -111,6 +120,12 @@ class StorageTestMixin(TestBase): """Make the cache entry of *request* unreadable for *storage*.""" raise NotImplementedError + def _downgrade_cache_entry( + self, storage: Any, spider: Spider, request: Request + ) -> None: + """Rewrite the cache entry of *request* as Scrapy 2.14 would have.""" + raise NotImplementedError + def test_storage(self): with self._storage(HTTPCACHE_EXPIRATION_SECS=100) as (storage, crawler): request2 = self.request.copy() @@ -184,6 +199,38 @@ class StorageTestMixin(TestBase): assert isinstance(cached_response, HtmlResponse) self.assertEqualResponse(response, cached_response) + def test_storage_response_class(self): + with self._storage() as (storage, crawler): + response = CustomResponse( + "http://www.example.com", body=b"test body", custom="value" + ) + storage.store_response(crawler.spider, self.request, response) + cached_response = storage.retrieve_response(crawler.spider, self.request) + assert isinstance(cached_response, CustomResponse) + assert cached_response.custom == "value" + + def test_storage_encoding(self): + """The encoding of the stored response is kept even when it cannot be + inferred from the response data.""" + with self._storage() as (storage, crawler): + response = HtmlResponse( + "http://www.example.com", + body='€'.encode(), + encoding="utf-8", + ) + storage.store_response(crawler.spider, self.request, response) + cached_response = storage.retrieve_response(crawler.spider, self.request) + assert cached_response.encoding == "utf-8" + assert cached_response.text == response.text + + def test_storage_old_cache_entry(self): + with self._storage() as (storage, crawler): + storage.store_response(crawler.spider, self.request, self.response) + self._downgrade_cache_entry(storage, crawler.spider, self.request) + cached_response = storage.retrieve_response(crawler.spider, self.request) + assert isinstance(cached_response, HtmlResponse) + self.assertEqualResponse(self.response, cached_response) + class PolicyTestMixin(TestBase): """Mixin containing policy-specific test methods.""" @@ -682,6 +729,10 @@ class FilesystemStorageTestMixin(StorageTestMixin): rpath = Path(storage._get_request_path(spider, request)) (rpath / "response_body").unlink() + def _downgrade_cache_entry(self, storage, spider, request) -> None: + rpath = Path(storage._get_request_path(spider, request)) + (rpath / "response_data").unlink() + class DbmStorageTestMixin(StorageTestMixin): storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" @@ -690,6 +741,14 @@ class DbmStorageTestMixin(StorageTestMixin): key = storage._fingerprinter.fingerprint(request).hex() storage.db[f"{key}_data"] = b"not a pickle" + def _downgrade_cache_entry(self, storage, spider, request) -> None: + key = storage._fingerprinter.fingerprint(request).hex() + data = pickle.loads(storage.db[f"{key}_data"]) + data = { + k: v for k, v in data.items() if k in ("status", "url", "headers", "body") + } + storage.db[f"{key}_data"] = pickle.dumps(data, protocol=4) + class TestFilesystemStorageWithDummyPolicy( FilesystemStorageTestMixin, DummyPolicyTestMixin diff --git a/tests/test_response_dict.py b/tests/test_response_dict.py new file mode 100644 index 000000000..e258a2316 --- /dev/null +++ b/tests/test_response_dict.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from ipaddress import ip_address + +from scrapy import Request +from scrapy.http import HtmlResponse, Response, TextResponse +from scrapy.utils.response import response_from_dict + + +class CustomResponse(TextResponse): + attributes: tuple[str, ...] = (*TextResponse.attributes, "custom") + + def __init__(self, *args, custom: str | None = None, **kwargs): + self.custom = custom + super().__init__(*args, **kwargs) + + +class DynamicResponse(Response): + """Response subclass that extends + :attr:`~scrapy.http.Response.attributes` on instances, as plugins that + support several Scrapy versions do.""" + + def __init__(self, *args, custom: str | None = None, **kwargs): + self.custom = custom + super().__init__(*args, **kwargs) + self.attributes = (*self.attributes, "custom") + + +def test_basic() -> None: + """The class of plain responses is not stored, it is guessed back from the + response data.""" + response = Response("https://example.com", body=b"\x00\x01") + assert "_class" not in response.to_dict() + response2 = response_from_dict(response.to_dict()) + assert response2.__class__ is Response + assert response2.url == response.url + + +def test_all_attributes() -> None: + response = HtmlResponse( + url="https://example.com", + status=201, + headers={"Content-Type": "text/html; charset=latin-1"}, + body=b"\xa3", + flags=["testFlag"], + encoding="latin-1", + ip_address=ip_address("127.0.0.1"), + protocol="h2", + ) + response2 = response_from_dict(response.to_dict()) + assert response2.__class__ is HtmlResponse + for attribute in HtmlResponse.attributes: + if attribute in {"request", "certificate"}: + continue + assert getattr(response2, attribute) == getattr(response, attribute) + + +def test_custom_attributes() -> None: + response = CustomResponse("https://example.com", custom="value") + response2 = response_from_dict(response.to_dict()) + assert isinstance(response2, CustomResponse) + assert response2.custom == "value" + + +def test_custom_instance_attributes() -> None: + response = DynamicResponse("https://example.com", custom="value") + response2 = response_from_dict(response.to_dict()) + assert isinstance(response2, DynamicResponse) + assert response2.custom == "value" + + +def test_crawl_attributes_left_out() -> None: + response = Response( + "https://example.com", + request=Request("https://example.com"), + certificate=object(), + ) + d = response.to_dict() + assert "request" not in d + assert "certificate" not in d + response2 = response_from_dict(d) + assert response2.request is None + assert response2.certificate is None + + +def test_unknown_class() -> None: + """Dicts that do not indicate a response class, e.g. cache entries written + by older Scrapy versions, get a class based on their data.""" + response2 = response_from_dict( + { + "url": "https://example.com", + "status": 200, + "headers": {b"Content-Type": [b"text/plain"]}, + "body": b"foo", + } + ) + assert response2.__class__ is TextResponse + assert response2.text == "foo"