diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index c702cefe7..b558c1cf2 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -121,6 +121,13 @@ one per actual value of the placeholder. :meth:`~scrapy.statscollectors.StatsCollector.get_stats` output is equivalent to a counter of 0. +.. stat:: depth/request_ignored_count + +``depth/request_ignored_count`` + Number of requests dropped for exceeding :setting:`DEPTH_LIMIT`. + + Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`. + .. stat:: downloader/exception_count ``downloader/exception_count`` diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 054804119..0131b62e7 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -41,6 +41,7 @@ class DepthMiddleware(BaseSpiderMiddleware): self.stats = stats self.verbose_stats = verbose_stats self.prio = prio + self._ignored_logged = False @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -94,11 +95,14 @@ class DepthMiddleware(BaseSpiderMiddleware): if self.prio: request.priority -= depth * self.prio if self.maxdepth and depth > self.maxdepth: - logger.debug( - "Ignoring link (depth > %(maxdepth)d): %(requrl)s ", - {"maxdepth": self.maxdepth, "requrl": request.url}, - extra={"spider": self.crawler.spider}, - ) + if not self._ignored_logged: + logger.debug( + f"Ignoring link (depth > {self.maxdepth}): {request.url}" + " - no more ignored links will be shown", + extra={"spider": self.crawler.spider}, + ) + self._ignored_logged = True + self.stats.inc_value("depth/request_ignored_count") return None if self.verbose_stats: self.stats.inc_value(f"request_depth_count/{depth}") diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index 32a2ea8f2..2aa76195e 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -85,6 +85,23 @@ async def test_process_spider_output_async_no_response( assert stats.get_value("request_depth_count/0") is None +def test_ignored_logged_once( + mw: DepthMiddleware, stats: StatsCollector, caplog: pytest.LogCaptureFixture +) -> None: + resp = Response("http://example.com") + resp.request = Request("http://example.com") + resp.meta["depth"] = 1 + result = [Request(f"http://example.com/{i}") for i in range(3)] + + with caplog.at_level("DEBUG", logger="scrapy.spidermiddlewares.depth"): + assert not list(mw.process_spider_output(resp, result)) + + messages = [r.getMessage() for r in caplog.records] + assert len(messages) == 1 + assert "http://example.com/0" in messages[0] + assert stats.get_value("depth/request_ignored_count") == 3 + + def test_priority_and_non_verbose_stats() -> None: crawler = get_crawler( Spider,