From f341b51cf2cf2eaf45527d185d4ccc0a8c982b03 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Sun, 9 Aug 2026 17:23:35 +0200 Subject: [PATCH 1/2] Add HTTPCACHE_SCOPE to allow sharing the HTTP cache across spiders --- docs/topics/downloader-middleware.rst | 42 ++++++++++-- docs/topics/request-response.rst | 4 +- scrapy/extensions/httpcache.py | 29 +++++++- scrapy/settings/default_settings.py | 2 + tests/test_downloadermiddleware_httpcache.py | 71 ++++++++++++++++++++ 5 files changed, 140 insertions(+), 8 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index fcfe7fd29..64a9bd747 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -516,9 +516,10 @@ Filesystem storage backend (default) The directory name is made from the request fingerprint (see ``scrapy.utils.request.fingerprint``), and one level of subdirectories is used to avoid creating too many files into the same directory (which is - inefficient in many file systems). An example directory could be:: + inefficient in many file systems). Those directories live inside the + folder that :setting:`HTTPCACHE_SCOPE` selects, ``my_spider`` below:: - /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7 + /path/to/cache/dir/my_spider/72/72811f648e718090f041317756c03adb0ada46c7 .. _httpcache-storage-dbm: @@ -626,9 +627,40 @@ HTTPCACHE_DIR Default: ``'httpcache'`` -The directory to use for storing the (low-level) HTTP cache. If empty, the HTTP -cache will be disabled. If a relative path is given, is taken relative to the -project data dir. For more info see: :ref:`topics-project-structure`. +The directory to use for storing the (low-level) HTTP cache. If a relative path +is given, is taken relative to the project data dir. For more info see: +:ref:`topics-project-structure`. + +.. setting:: HTTPCACHE_SCOPE + +HTTPCACHE_SCOPE +^^^^^^^^^^^^^^^ + +Default: ``'spider'`` + +.. versionadded:: VERSION + +How to partition the cache within :setting:`HTTPCACHE_DIR`: + +``'spider'`` + One cache per spider. Spiders never reuse each other's cached responses. + +``'none'`` + A single cache, shared by every spider. + +``'domain'`` + One cache per request host, shared by every spider. Only + :class:`~scrapy.extensions.httpcache.FilesystemCacheStorage` supports this + value; it allows removing the cached responses of a single website by + deleting its directory. + +Changing this setting makes existing cached responses unreachable, since +nothing looks for them in their old location. Delete +:setting:`HTTPCACHE_DIR` to reclaim their disk space. + +With :class:`~scrapy.extensions.httpcache.DbmCacheStorage`, ``'none'`` makes +every spider use the same database file, which most DBM implementations open +for exclusive writing. Spiders that share a cache must then run one at a time. .. setting:: HTTPCACHE_IGNORE_HTTP_CODES diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a177d1ad5..5b8cddf98 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -526,7 +526,9 @@ The following built-in Scrapy components have such restrictions: :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`, the following directory structure is created: - - :attr:`.Spider.name` + - the partition that :setting:`HTTPCACHE_SCOPE` selects, i.e. + :attr:`.Spider.name` or the request host, and no folder at all with + ``'none'`` - first byte of a request fingerprint as hexadecimal diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index dbb79b02d..928bec26b 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -8,6 +8,7 @@ from importlib import import_module from pathlib import Path from time import time from typing import IO, TYPE_CHECKING, Any, Concatenate, cast +from urllib.parse import quote from weakref import WeakKeyDictionary from w3lib.http import headers_dict_to_raw, headers_raw_to_dict @@ -32,6 +33,17 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +def _get_scope(settings: BaseSettings, supported: tuple[str, ...]) -> str: + scope: str = settings["HTTPCACHE_SCOPE"] + if scope not in supported: + supported_repr = ", ".join(repr(value) for value in supported) + raise ValueError( + f"Unsupported HTTPCACHE_SCOPE value: {scope!r}. The cache storage " + f"in use supports the following values: {supported_repr}." + ) + return scope + + class DummyPolicy: def __init__(self, settings: BaseSettings): self.ignore_schemes: list[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES") @@ -250,9 +262,11 @@ class DbmCacheStorage: self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") self.dbmodule: ModuleType = import_module(settings["HTTPCACHE_DBM_MODULE"]) self.db: Any = None # the real type is private + self._scope = _get_scope(settings, ("spider", "none")) def open_spider(self, spider: Spider) -> None: - dbpath = Path(self.cachedir, f"{spider.name}.db") + name = spider.name if self._scope == "spider" else "httpcache" + dbpath = Path(self.cachedir, f"{name}.db") self.db = self.dbmodule.open(str(dbpath), "c") logger.debug( @@ -312,6 +326,7 @@ class FilesystemCacheStorage: self.cachedir: str = data_path(settings["HTTPCACHE_DIR"]) self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") + self._scope = _get_scope(settings, ("spider", "none", "domain")) # https://github.com/python/mypy/issues/10740 self._open: Callable[ Concatenate[str | os.PathLike[str], str, ...], IO[bytes] @@ -377,7 +392,17 @@ class FilesystemCacheStorage: def _get_request_path(self, spider: Spider, request: Request) -> str: key = self._fingerprinter.fingerprint(request).hex() - return str(Path(self.cachedir, spider.name, key[0:2], key)) + scope_path = self._get_scope_path(spider, request) + return str(Path(self.cachedir, scope_path, key[0:2], key)) + + def _get_scope_path(self, spider: Spider, request: Request) -> str: + if self._scope == "spider": + return spider.name + if self._scope == "domain": + # quote() covers hostnames that are not valid path components, + # such as IPv6 addresses. + return quote(urlparse_cached(request).hostname or "_", safe="") + return "" def _read_meta(self, spider: Spider, request: Request) -> dict[str, Any] | None: rpath = Path(self._get_request_path(spider, request)) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a44b36c8a..39c1726e5 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -122,6 +122,7 @@ __all__ = [ "HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS", "HTTPCACHE_IGNORE_SCHEMES", "HTTPCACHE_POLICY", + "HTTPCACHE_SCOPE", "HTTPCACHE_STORAGE", "HTTPERROR_ALLOWED_CODES", "HTTPERROR_ALLOW_ALL", @@ -416,6 +417,7 @@ HTTPCACHE_IGNORE_MISSING = False HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = [] HTTPCACHE_IGNORE_SCHEMES = ["file"] HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy" +HTTPCACHE_SCOPE = "spider" HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" HTTPERROR_ALLOW_ALL = False diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index dc8228470..0c2635b76 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -17,6 +17,7 @@ from scrapy.exceptions import IgnoreRequest from scrapy.extensions.httpcache import DummyPolicy from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider +from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler if TYPE_CHECKING: @@ -732,3 +733,73 @@ class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPol # A spider killed while writing a gzip file leaves it truncated. body_path = Path(storage._get_request_path(spider, request), "response_body") body_path.write_bytes(body_path.read_bytes()[:-5]) + + +class ScopeTestMixin(TestBase): + policy_class = "scrapy.extensions.httpcache.DummyPolicy" + + @contextmanager + def _spider_storage(self, crawler: Crawler, name: str) -> Generator[Any]: + spider = crawler._create_spider(name) + storage = load_object(self.storage_class)(crawler.settings) + storage.open_spider(spider) + try: + yield storage, spider + finally: + storage.close_spider(spider) + + def _cross_spider_retrieval(self, scope: str) -> Any: + # Storages are opened one at a time because a DBM database shared by + # two spiders does not support concurrent access. + with self._get_crawler(HTTPCACHE_SCOPE=scope) as crawler: + with self._spider_storage(crawler, "a") as (storage, spider): + storage.store_response(spider, self.request, self.response) + with self._spider_storage(crawler, "b") as (storage, spider): + return storage.retrieve_response(spider, self.request) + + def test_spider_scope(self): + assert self._cross_spider_retrieval("spider") is None + + def test_no_scope(self): + assert self._cross_spider_retrieval("none") is not None + + def test_unknown_scope(self): + with ( + self._get_crawler(HTTPCACHE_SCOPE="unknown") as crawler, + pytest.raises(ValueError, match="Unsupported HTTPCACHE_SCOPE"), + ): + load_object(self.storage_class)(crawler.settings) + + +class TestFilesystemStorageScope(ScopeTestMixin): + storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage" + + def test_domain_scope(self): + assert self._cross_spider_retrieval("domain") is not None + + @pytest.mark.parametrize( + ("url", "expected"), + [ + ("http://user:pass@WWW.Example.com:8080/", "www.example.com"), + ("http://[::1]/", "%3A%3A1"), + ("file:///tmp/t.txt", "_"), + ], + ) + def test_domain_scope_path(self, url, expected): + with ( + self._get_crawler(HTTPCACHE_SCOPE="domain") as crawler, + self._spider_storage(crawler, "a") as (storage, spider), + ): + path = Path(storage._get_request_path(spider, Request(url))) + assert path.relative_to(self.tmpdir).parts[0] == expected + + +class TestDbmStorageScope(ScopeTestMixin): + storage_class = "scrapy.extensions.httpcache.DbmCacheStorage" + + def test_domain_scope_unsupported(self): + with ( + self._get_crawler(HTTPCACHE_SCOPE="domain") as crawler, + pytest.raises(ValueError, match="Unsupported HTTPCACHE_SCOPE"), + ): + load_object(self.storage_class)(crawler.settings) From 829462020b9a2dde48db6d03cc5d3e8ce7c1b247 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Sun, 9 Aug 2026 17:55:27 +0200 Subject: [PATCH 2/2] Skip test on bad w3lib --- tests/test_downloadermiddleware_httpcache.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 0c2635b76..fd6b74f47 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -9,6 +9,7 @@ from contextlib import contextmanager from pathlib import Path from typing import TYPE_CHECKING, Any from unittest import mock +from urllib.parse import urlparse import pytest @@ -781,7 +782,14 @@ class TestFilesystemStorageScope(ScopeTestMixin): ("url", "expected"), [ ("http://user:pass@WWW.Example.com:8080/", "www.example.com"), - ("http://[::1]/", "%3A%3A1"), + pytest.param( + "http://[::1]/", + "%3A%3A1", + marks=pytest.mark.skipif( + urlparse(Request("http://[::1]/").url).hostname != "::1", + reason="w3lib strips the brackets of IPv6 hosts", + ), + ), ("file:///tmp/t.txt", "_"), ], )