Add a depth_reset request metadata key (#7913)

docs/topics/spider-middleware.rst

@@ -228,25 +228,7 @@ DepthMiddleware
 .. module:: scrapy.spidermiddlewares.depth
    :synopsis: Depth Spider Middleware
 
-.. class:: DepthMiddleware
-
-   DepthMiddleware is used for tracking the depth of each Request inside the
-   site being scraped. It works by setting ``request.meta['depth'] = 0`` whenever
-   there is no value previously set (usually just the first Request) and
-   incrementing it by 1 otherwise.
-
-   It can be used to limit the maximum depth to scrape, control Request
-   priority based on their depth, and things like that.
-
-   The :class:`DepthMiddleware` can be configured through the following
-   settings (see the settings documentation for more info):
-
-      * :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to
-        crawl for any site. If zero, no limit will be imposed.
-      * :setting:`DEPTH_STATS_VERBOSE` - Whether to collect the number of
-        requests for each depth.
-      * :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on
-        their depth.
+.. autoclass:: DepthMiddleware
 
 HttpErrorMiddleware
 -------------------

scrapy/spidermiddlewares/depth.py

@@ -28,6 +28,27 @@ logger = logging.getLogger(__name__)
 
 
 class DepthMiddleware(BaseSpiderMiddleware):
+    """Track the depth of each request within the site being scraped, setting
+    ``request.meta["depth"]`` to 0 when there is no value previously set
+    (usually just the first request) and incrementing it by 1 otherwise.
+
+    It can be used to limit the maximum depth to scrape, control request
+    priority based on their depth, and things like that, through the
+    :setting:`DEPTH_LIMIT`, :setting:`DEPTH_STATS_VERBOSE` and
+    :setting:`DEPTH_PRIORITY` settings.
+
+    .. reqmeta:: depth_reset
+
+    depth_reset
+    -----------
+
+    .. versionadded:: VERSION
+
+    :attr:`~scrapy.Request.meta` key that, set to ``True``, gives a request
+    depth 0 instead of the depth of its source response plus 1, e.g. to keep
+    :setting:`DEPTH_LIMIT` from applying across a domain change.
+    """
+
     crawler: Crawler
 
     def __init__(  # pylint: disable=super-init-not-called
@@ -87,10 +108,13 @@ class DepthMiddleware(BaseSpiderMiddleware):
     def get_processed_request(
         self, request: Request, response: Response | None
     ) -> Request | None:
+        # Consumed here so that it cannot reach response.meta and, from there,
+        # spread to further requests through a meta copy.
+        depth_reset = request.meta.pop("depth_reset", False)
         if response is None:
             # start requests
             return request
-        depth = response.meta["depth"] + 1
+        depth = 0 if depth_reset else response.meta["depth"] + 1
         request.meta["depth"] = depth
         if self.prio:
             request.priority -= depth * self.prio

tests/test_spidermiddleware_depth.py

@@ -60,6 +60,19 @@ def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> No
     assert rdm == 1
 
 
+def test_depth_reset(mw: DepthMiddleware, stats: StatsCollector) -> None:
+    resp = Response("https://example.com")
+    resp.request = Request("https://example.com", meta={"depth": 5})
+    result = [Request("https://example.com", meta={"depth_reset": True})]
+
+    out = list(mw.process_spider_output(resp, result))
+
+    assert out == result
+    assert out[0].meta["depth"] == 0
+    assert "depth_reset" not in out[0].meta
+    assert stats.get_value("request_depth_count/0") == 1
+
+
 def test_process_spider_output_no_response(
     mw: DepthMiddleware, stats: StatsCollector
 ) -> None: