Merge 780ab92345 into ad43bf0c56

docs/topics/downloader-middleware.rst

@@ -475,9 +475,10 @@ Filesystem storage backend (default)
     The directory name is made from the request fingerprint (see
     ``scrapy.utils.request.fingerprint``), and one level of subdirectories is
     used to avoid creating too many files into the same directory (which is
-    inefficient in many file systems). An example directory could be::
+    inefficient in many file systems). Those directories live inside the
+    folder that :setting:`HTTPCACHE_SCOPE` selects, ``my_spider`` below::
 
-        /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7
+        /path/to/cache/dir/my_spider/72/72811f648e718090f041317756c03adb0ada46c7
 
 .. _httpcache-storage-dbm:
 
@@ -585,9 +586,40 @@ HTTPCACHE_DIR
 
 Default: ``'httpcache'``
 
-The directory to use for storing the (low-level) HTTP cache. If empty, the HTTP
-cache will be disabled. If a relative path is given, is taken relative to the
-project data dir. For more info see: :ref:`topics-project-structure`.
+The directory to use for storing the (low-level) HTTP cache. If a relative path
+is given, is taken relative to the project data dir. For more info see:
+:ref:`topics-project-structure`.
+
+.. setting:: HTTPCACHE_SCOPE
+
+HTTPCACHE_SCOPE
+^^^^^^^^^^^^^^^
+
+Default: ``'spider'``
+
+.. versionadded:: VERSION
+
+How to partition the cache within :setting:`HTTPCACHE_DIR`:
+
+``'spider'``
+    One cache per spider. Spiders never reuse each other's cached responses.
+
+``'none'``
+    A single cache, shared by every spider.
+
+``'domain'``
+    One cache per request host, shared by every spider. Only
+    :class:`~scrapy.extensions.httpcache.FilesystemCacheStorage` supports this
+    value; it allows removing the cached responses of a single website by
+    deleting its directory.
+
+Changing this setting makes existing cached responses unreachable, since
+nothing looks for them in their old location. Delete
+:setting:`HTTPCACHE_DIR` to reclaim their disk space.
+
+With :class:`~scrapy.extensions.httpcache.DbmCacheStorage`, ``'none'`` makes
+every spider use the same database file, which most DBM implementations open
+for exclusive writing. Spiders that share a cache must then run one at a time.
 
 .. setting:: HTTPCACHE_IGNORE_HTTP_CODES
 

docs/topics/request-response.rst

@@ -474,7 +474,9 @@ The following built-in Scrapy components have such restrictions:
     :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
     the following directory structure is created:
 
-    -   :attr:`.Spider.name`
+    -   the partition that :setting:`HTTPCACHE_SCOPE` selects, i.e.
+        :attr:`.Spider.name` or the request host, and no folder at all with
+        ``'none'``
 
         -   first byte of a request fingerprint as hexadecimal
 

scrapy/extensions/httpcache.py

@@ -8,6 +8,7 @@ from importlib import import_module
 from pathlib import Path
 from time import time
 from typing import IO, TYPE_CHECKING, Any, Concatenate, cast
+from urllib.parse import quote
 from weakref import WeakKeyDictionary
 
 from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
@@ -32,6 +33,17 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+def _get_scope(settings: BaseSettings, supported: tuple[str, ...]) -> str:
+    scope: str = settings["HTTPCACHE_SCOPE"]
+    if scope not in supported:
+        supported_repr = ", ".join(repr(value) for value in supported)
+        raise ValueError(
+            f"Unsupported HTTPCACHE_SCOPE value: {scope!r}. The cache storage "
+            f"in use supports the following values: {supported_repr}."
+        )
+    return scope
+
+
 class DummyPolicy:
     def __init__(self, settings: BaseSettings):
         self.ignore_schemes: list[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES")
@@ -250,9 +262,11 @@ class DbmCacheStorage:
         self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")
         self.dbmodule: ModuleType = import_module(settings["HTTPCACHE_DBM_MODULE"])
         self.db: Any = None  # the real type is private
+        self._scope = _get_scope(settings, ("spider", "none"))
 
     def open_spider(self, spider: Spider) -> None:
-        dbpath = Path(self.cachedir, f"{spider.name}.db")
+        name = spider.name if self._scope == "spider" else "httpcache"
+        dbpath = Path(self.cachedir, f"{name}.db")
         self.db = self.dbmodule.open(str(dbpath), "c")
 
         logger.debug(
@@ -311,6 +325,7 @@ class FilesystemCacheStorage:
         self.cachedir: str = data_path(settings["HTTPCACHE_DIR"])
         self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")
         self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP")
+        self._scope = _get_scope(settings, ("spider", "none", "domain"))
         # https://github.com/python/mypy/issues/10740
         self._open: Callable[
             Concatenate[str | os.PathLike[str], str, ...], IO[bytes]
@@ -375,7 +390,17 @@ class FilesystemCacheStorage:
 
     def _get_request_path(self, spider: Spider, request: Request) -> str:
         key = self._fingerprinter.fingerprint(request).hex()
-        return str(Path(self.cachedir, spider.name, key[0:2], key))
+        scope_path = self._get_scope_path(spider, request)
+        return str(Path(self.cachedir, scope_path, key[0:2], key))
+
+    def _get_scope_path(self, spider: Spider, request: Request) -> str:
+        if self._scope == "spider":
+            return spider.name
+        if self._scope == "domain":
+            # quote() covers hostnames that are not valid path components,
+            # such as IPv6 addresses.
+            return quote(urlparse_cached(request).hostname or "_", safe="")
+        return ""
 
     def _read_meta(self, spider: Spider, request: Request) -> dict[str, Any] | None:
         rpath = Path(self._get_request_path(spider, request))

scrapy/settings/default_settings.py

@@ -123,6 +123,7 @@ __all__ = [
     "HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS",
     "HTTPCACHE_IGNORE_SCHEMES",
     "HTTPCACHE_POLICY",
+    "HTTPCACHE_SCOPE",
     "HTTPCACHE_STORAGE",
     "HTTPERROR_ALLOWED_CODES",
     "HTTPERROR_ALLOW_ALL",
@@ -420,6 +421,7 @@ HTTPCACHE_IGNORE_MISSING = False
 HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
 HTTPCACHE_IGNORE_SCHEMES = ["file"]
 HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"
+HTTPCACHE_SCOPE = "spider"
 HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
 
 HTTPERROR_ALLOW_ALL = False

tests/test_downloadermiddleware_httpcache.py

@@ -9,6 +9,7 @@ from contextlib import contextmanager
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 from unittest import mock
+from urllib.parse import urlparse
 
 import pytest
 
@@ -17,7 +18,7 @@ from scrapy.exceptions import IgnoreRequest
 from scrapy.extensions.httpcache import DummyPolicy
 from scrapy.http import HtmlResponse, Request, Response
 from scrapy.spiders import Spider
-from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.test import get_crawler
 
 if TYPE_CHECKING:
@@ -733,3 +734,80 @@ class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPol
         # A spider killed while writing a gzip file leaves it truncated.
         body_path = Path(storage._get_request_path(spider, request), "response_body")
         body_path.write_bytes(body_path.read_bytes()[:-5])
+
+
+class ScopeTestMixin(TestBase):
+    policy_class = "scrapy.extensions.httpcache.DummyPolicy"
+
+    @contextmanager
+    def _spider_storage(self, crawler: Crawler, name: str) -> Generator[Any]:
+        spider = crawler._create_spider(name)
+        storage = load_object(self.storage_class)(crawler.settings)
+        storage.open_spider(spider)
+        try:
+            yield storage, spider
+        finally:
+            storage.close_spider(spider)
+
+    def _cross_spider_retrieval(self, scope: str) -> Any:
+        # Storages are opened one at a time because a DBM database shared by
+        # two spiders does not support concurrent access.
+        with self._get_crawler(HTTPCACHE_SCOPE=scope) as crawler:
+            with self._spider_storage(crawler, "a") as (storage, spider):
+                storage.store_response(spider, self.request, self.response)
+            with self._spider_storage(crawler, "b") as (storage, spider):
+                return storage.retrieve_response(spider, self.request)
+
+    def test_spider_scope(self):
+        assert self._cross_spider_retrieval("spider") is None
+
+    def test_no_scope(self):
+        assert self._cross_spider_retrieval("none") is not None
+
+    def test_unknown_scope(self):
+        with (
+            self._get_crawler(HTTPCACHE_SCOPE="unknown") as crawler,
+            pytest.raises(ValueError, match="Unsupported HTTPCACHE_SCOPE"),
+        ):
+            load_object(self.storage_class)(crawler.settings)
+
+
+class TestFilesystemStorageScope(ScopeTestMixin):
+    storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
+
+    def test_domain_scope(self):
+        assert self._cross_spider_retrieval("domain") is not None
+
+    @pytest.mark.parametrize(
+        ("url", "expected"),
+        [
+            ("http://user:pass@WWW.Example.com:8080/", "www.example.com"),
+            pytest.param(
+                "http://[::1]/",
+                "%3A%3A1",
+                marks=pytest.mark.skipif(
+                    urlparse(Request("http://[::1]/").url).hostname != "::1",
+                    reason="w3lib strips the brackets of IPv6 hosts",
+                ),
+            ),
+            ("file:///tmp/t.txt", "_"),
+        ],
+    )
+    def test_domain_scope_path(self, url, expected):
+        with (
+            self._get_crawler(HTTPCACHE_SCOPE="domain") as crawler,
+            self._spider_storage(crawler, "a") as (storage, spider),
+        ):
+            path = Path(storage._get_request_path(spider, Request(url)))
+            assert path.relative_to(self.tmpdir).parts[0] == expected
+
+
+class TestDbmStorageScope(ScopeTestMixin):
+    storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
+
+    def test_domain_scope_unsupported(self):
+        with (
+            self._get_crawler(HTTPCACHE_SCOPE="domain") as crawler,
+            pytest.raises(ValueError, match="Unsupported HTTPCACHE_SCOPE"),
+        ):
+            load_object(self.storage_class)(crawler.settings)