Serialize cached responses with Response.to_dict()

docs/topics/downloader-middleware.rst

@@ -507,6 +507,9 @@ Filesystem storage backend (default)
 
     *   ``response_headers`` - the response headers (in raw HTTP format)
 
+    *   ``response_data`` - the remaining data of the response, as returned by
+        :meth:`Response.to_dict() <scrapy.http.Response.to_dict>`, pickled
+
     *   ``meta`` - some metadata of this cache resource in Python ``repr()``
         format (grep-friendly format)
 
@@ -540,6 +543,11 @@ Writing your own storage backend
 You can implement a cache storage backend by creating a Python class that
 defines the methods described below.
 
+To store a response, use :meth:`Response.to_dict()
+<scrapy.http.Response.to_dict>`, and to read it back, use
+:func:`~scrapy.utils.response.response_from_dict`. That way responses of any
+class, including those of third-party plugins, are cached and restored intact.
+
 .. module:: scrapy.extensions.httpcache
 
 .. class:: CacheStorage

docs/topics/request-response.rst

@@ -1315,6 +1315,10 @@ Response objects
 
     .. automethod:: Response.follow_all
 
+    .. automethod:: Response.to_dict
+
+    .. automethod:: Response.from_dict
+
 
 .. _topics-request-response-ref-response-subclasses:
 
@@ -1471,3 +1475,9 @@ JsonResponse objects
     that is used when the response has a `JSON MIME type
     <https://mimesniff.spec.whatwg.org/#json-mime-type>`_ in its `Content-Type`
     header.
+
+
+Other functions related to responses
+====================================
+
+.. autofunction:: scrapy.utils.response.response_from_dict

scrapy/extensions/httpcache.py

@@ -12,11 +12,11 @@ from weakref import WeakKeyDictionary
 
 from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
 
-from scrapy.http import Headers, Response
-from scrapy.responsetypes import responsetypes
+from scrapy.http import Response
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.project import data_path
 from scrapy.utils.python import to_bytes, to_unicode
+from scrapy.utils.response import response_from_dict
 
 if TYPE_CHECKING:
     import os
@@ -273,24 +273,13 @@ class DbmCacheStorage:
         data = self._read_data(spider, request)
         if data is None:
             return None  # not cached
-        url = data["url"]
-        status = data["status"]
-        headers = Headers(data["headers"])
-        body = data["body"]
-        respcls = responsetypes.from_args(headers=headers, url=url, body=body)
-        return respcls(url=url, headers=headers, status=status, body=body)
+        return response_from_dict(data)
 
     def store_response(
         self, spider: Spider, request: Request, response: Response
     ) -> None:
         key = self._fingerprinter.fingerprint(request).hex()
-        data = {
-            "status": response.status,
-            "url": response.url,
-            "headers": dict(response.headers),
-            "body": response.body,
-        }
-        self.db[f"{key}_data"] = pickle.dumps(data, protocol=4)
+        self.db[f"{key}_data"] = pickle.dumps(response.to_dict(), protocol=4)
         self.db[f"{key}_time"] = str(time())
 
     def _read_data(self, spider: Spider, request: Request) -> dict[str, Any] | None:
@@ -342,11 +331,17 @@ class FilesystemCacheStorage:
             body = f.read()
         with self._open(rpath / "response_headers", "rb") as f:
             rawheaders = f.read()
-        url = metadata["response_url"]
-        status = metadata["status"]
-        headers = Headers(headers_raw_to_dict(rawheaders))
-        respcls = responsetypes.from_args(headers=headers, url=url, body=body)
-        return respcls(url=url, headers=headers, status=status, body=body)
+        data = {
+            "url": metadata["response_url"],
+            "status": metadata["status"],
+            "headers": headers_raw_to_dict(rawheaders),
+            "body": body,
+        }
+        datapath = rpath / "response_data"
+        if datapath.exists():
+            with self._open(datapath, "rb") as f:
+                data.update(pickle.load(f))  # noqa: S301
+        return response_from_dict(data)
 
     def store_response(
         self, spider: Spider, request: Request, response: Response
@@ -370,6 +365,13 @@ class FilesystemCacheStorage:
             f.write(headers_dict_to_raw(response.headers))
         with self._open(rpath / "response_body", "wb") as f:
             f.write(response.body)
+        data = {
+            key: value
+            for key, value in response.to_dict().items()
+            if key not in {"url", "status", "headers", "body"}
+        }
+        with self._open(rpath / "response_data", "wb") as f:
+            pickle.dump(data, f, protocol=4)
         with self._open(rpath / "request_headers", "wb") as f:
             f.write(headers_dict_to_raw(request.headers))
         with self._open(rpath / "request_body", "wb") as f:

scrapy/http/response/__init__.py

@@ -194,6 +194,37 @@ class Response(object_ref):
             cls = self.__class__
         return cls(*args, **kwargs)
 
+    def to_dict(self) -> dict[str, Any]:
+        """Return a dictionary containing the Response's data.
+
+        .. versionadded:: VERSION
+
+        Use :func:`~scrapy.utils.response.response_from_dict` to convert back
+        into a :class:`~scrapy.http.Response` object.
+
+        :attr:`request` and :attr:`certificate` are left out, as they are tied
+        to a single crawl. Everything else in :attr:`attributes` is included,
+        so subclasses only need to override this method, and :meth:`from_dict`,
+        if some of their attributes cannot be stored as is.
+        """
+        d: dict[str, Any] = {"headers": dict(self.headers)}
+        for attr in self.attributes:
+            if attr in {"request", "certificate"}:
+                continue
+            d.setdefault(attr, getattr(self, attr))
+        if type(self) is not Response:  # pylint: disable=unidiomatic-typecheck
+            d["_class"] = self.__module__ + "." + self.__class__.__name__
+        return d
+
+    @classmethod
+    def from_dict(cls, d: dict[str, Any]) -> Self:
+        """Return a response built from the *d* dict, as returned by
+        :meth:`to_dict`.
+
+        .. versionadded:: VERSION
+        """
+        return cls(**{key: value for key, value in d.items() if key != "_class"})
+
     def urljoin(self, url: str) -> str:
         """Join this Response's url with a possible relative url to form an
         absolute interpretation of the latter."""

scrapy/utils/response.py

@@ -15,6 +15,8 @@ from weakref import WeakKeyDictionary
 from twisted.web import http
 from w3lib import html
 
+from scrapy.http.headers import Headers
+from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes, to_unicode
 
 if TYPE_CHECKING:
@@ -53,6 +55,30 @@ def get_meta_refresh(
     return _metaref_cache[response]
 
 
+def response_from_dict(d: dict[str, Any]) -> Response:
+    """Return a response built from the *d* dict, as returned by
+    :meth:`Response.to_dict() <scrapy.http.Response.to_dict>`.
+
+    .. versionadded:: VERSION
+
+    If *d* does not indicate a response class, e.g. because it comes from a
+    plain :class:`~scrapy.http.Response` object or predates :meth:`~scrapy.http.Response.to_dict`,
+    the class is guessed with :attr:`~scrapy.responsetypes.responsetypes`.
+    """
+    # Imported here to avoid a circular import.
+    from scrapy.responsetypes import responsetypes  # noqa: PLC0415
+
+    d = {**d, "headers": Headers(d.get("headers") or {})}
+    response_cls: type[Response] = (
+        load_object(d["_class"])
+        if "_class" in d
+        else responsetypes.from_args(
+            headers=d["headers"], url=d["url"], body=d.get("body")
+        )
+    )
+    return response_cls.from_dict(d)
+
+
 def response_status_message(status: bytes | float | str) -> str:
     """Return status code plus status text descriptive message"""
     status_int = int(status)

tests/test_downloadermiddleware_httpcache.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import email.utils
 import logging
+import pickle
 import shutil
 import tempfile
 import time
@@ -25,6 +26,14 @@ if TYPE_CHECKING:
     from scrapy.crawler import Crawler
 
 
+class CustomResponse(Response):
+    attributes: tuple[str, ...] = (*Response.attributes, "custom")
+
+    def __init__(self, *args: Any, custom: str | None = None, **kwargs: Any):
+        self.custom = custom
+        super().__init__(*args, **kwargs)
+
+
 class AlwaysStalePolicy(DummyPolicy):
     """:class:`~scrapy.extensions.httpcache.DummyPolicy` that always
     revalidates cached responses."""
@@ -110,6 +119,12 @@ class StorageTestMixin(TestBase):
         """Make the cache entry of *request* unreadable for *storage*."""
         raise NotImplementedError
 
+    def _downgrade_cache_entry(
+        self, storage: Any, spider: Spider, request: Request
+    ) -> None:
+        """Rewrite the cache entry of *request* as Scrapy 2.14 would have."""
+        raise NotImplementedError
+
     def test_storage(self):
         with self._storage(HTTPCACHE_EXPIRATION_SECS=1) as (storage, crawler):
             request2 = self.request.copy()
@@ -183,6 +198,38 @@ class StorageTestMixin(TestBase):
             assert isinstance(cached_response, HtmlResponse)
             self.assertEqualResponse(response, cached_response)
 
+    def test_storage_response_class(self):
+        with self._storage() as (storage, crawler):
+            response = CustomResponse(
+                "http://www.example.com", body=b"test body", custom="value"
+            )
+            storage.store_response(crawler.spider, self.request, response)
+            cached_response = storage.retrieve_response(crawler.spider, self.request)
+            assert isinstance(cached_response, CustomResponse)
+            assert cached_response.custom == "value"
+
+    def test_storage_encoding(self):
+        """The encoding of the stored response is kept even when it cannot be
+        inferred from the response data."""
+        with self._storage() as (storage, crawler):
+            response = HtmlResponse(
+                "http://www.example.com",
+                body='<meta charset="iso-8859-1">€'.encode(),
+                encoding="utf-8",
+            )
+            storage.store_response(crawler.spider, self.request, response)
+            cached_response = storage.retrieve_response(crawler.spider, self.request)
+            assert cached_response.encoding == "utf-8"
+            assert cached_response.text == response.text
+
+    def test_storage_old_cache_entry(self):
+        with self._storage() as (storage, crawler):
+            storage.store_response(crawler.spider, self.request, self.response)
+            self._downgrade_cache_entry(storage, crawler.spider, self.request)
+            cached_response = storage.retrieve_response(crawler.spider, self.request)
+            assert isinstance(cached_response, HtmlResponse)
+            self.assertEqualResponse(self.response, cached_response)
+
 
 class PolicyTestMixin(TestBase):
     """Mixin containing policy-specific test methods."""
@@ -681,6 +728,10 @@ class FilesystemStorageTestMixin(StorageTestMixin):
         rpath = Path(storage._get_request_path(spider, request))
         (rpath / "response_body").unlink()
 
+    def _downgrade_cache_entry(self, storage, spider, request) -> None:
+        rpath = Path(storage._get_request_path(spider, request))
+        (rpath / "response_data").unlink()
+
 
 class DbmStorageTestMixin(StorageTestMixin):
     storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
@@ -689,6 +740,14 @@ class DbmStorageTestMixin(StorageTestMixin):
         key = storage._fingerprinter.fingerprint(request).hex()
         storage.db[f"{key}_data"] = b"not a pickle"
 
+    def _downgrade_cache_entry(self, storage, spider, request) -> None:
+        key = storage._fingerprinter.fingerprint(request).hex()
+        data = pickle.loads(storage.db[f"{key}_data"])
+        data = {
+            k: v for k, v in data.items() if k in ("status", "url", "headers", "body")
+        }
+        storage.db[f"{key}_data"] = pickle.dumps(data, protocol=4)
+
 
 class TestFilesystemStorageWithDummyPolicy(
     FilesystemStorageTestMixin, DummyPolicyTestMixin

tests/test_response_dict.py

@@ -0,0 +1,98 @@
+from __future__ import annotations
+
+from ipaddress import ip_address
+
+from scrapy import Request
+from scrapy.http import HtmlResponse, Response, TextResponse
+from scrapy.utils.response import response_from_dict
+
+
+class CustomResponse(TextResponse):
+    attributes: tuple[str, ...] = (*TextResponse.attributes, "custom")
+
+    def __init__(self, *args, custom: str | None = None, **kwargs):
+        self.custom = custom
+        super().__init__(*args, **kwargs)
+
+
+class DynamicResponse(Response):
+    """Response subclass that extends
+    :attr:`~scrapy.http.Response.attributes` on instances, as plugins that
+    support several Scrapy versions do."""
+
+    def __init__(self, *args, custom: str | None = None, **kwargs):
+        self.custom = custom
+        super().__init__(*args, **kwargs)
+        self.attributes = (*self.attributes, "custom")
+
+
+def test_basic() -> None:
+    """The class of plain responses is not stored, it is guessed back from the
+    response data."""
+    response = Response("https://example.com", body=b"\x00\x01")
+    assert "_class" not in response.to_dict()
+    response2 = response_from_dict(response.to_dict())
+    assert response2.__class__ is Response
+    assert response2.url == response.url
+
+
+def test_all_attributes() -> None:
+    response = HtmlResponse(
+        url="https://example.com",
+        status=201,
+        headers={"Content-Type": "text/html; charset=latin-1"},
+        body=b"\xa3",
+        flags=["testFlag"],
+        encoding="latin-1",
+        ip_address=ip_address("127.0.0.1"),
+        protocol="h2",
+    )
+    response2 = response_from_dict(response.to_dict())
+    assert response2.__class__ is HtmlResponse
+    for attribute in HtmlResponse.attributes:
+        if attribute in {"request", "certificate"}:
+            continue
+        assert getattr(response2, attribute) == getattr(response, attribute)
+
+
+def test_custom_attributes() -> None:
+    response = CustomResponse("https://example.com", custom="value")
+    response2 = response_from_dict(response.to_dict())
+    assert isinstance(response2, CustomResponse)
+    assert response2.custom == "value"
+
+
+def test_custom_instance_attributes() -> None:
+    response = DynamicResponse("https://example.com", custom="value")
+    response2 = response_from_dict(response.to_dict())
+    assert isinstance(response2, DynamicResponse)
+    assert response2.custom == "value"
+
+
+def test_crawl_attributes_left_out() -> None:
+    response = Response(
+        "https://example.com",
+        request=Request("https://example.com"),
+        certificate=object(),
+    )
+    d = response.to_dict()
+    assert "request" not in d
+    assert "certificate" not in d
+    response2 = response_from_dict(d)
+    assert response2.request is None
+    assert response2.certificate is None
+
+
+def test_unknown_class() -> None:
+    """Dicts that do not indicate a response class, e.g. cache entries written
+    by older Scrapy versions, get a class based on their data."""
+    response2 = response_from_dict(
+        {
+            "url": "https://example.com",
+            "status": 200,
+            "headers": {b"Content-Type": [b"text/plain"]},
+            "body": b"foo",
+        }
+    )
+    assert response2.__class__ is TextResponse
+    assert response2.text == "foo"