From c7b90c6e1e3de20256d8bf6fc5d18da44d562b0d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 28 Jul 2022 13:44:36 +0500 Subject: [PATCH] Extract more common code. --- scrapy/spidermiddlewares/depth.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 29634c3ad..4c923b1b3 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -28,25 +28,22 @@ class DepthMiddleware: return cls(maxdepth, crawler.stats, verbose, prio) def process_spider_output(self, response, result, spider): - # base case (depth=0) - if 'depth' not in response.meta: - response.meta['depth'] = 0 - if self.verbose_stats: - self.stats.inc_value('request_depth_count/0', spider=spider) - + self._init_depth(response, spider) return (r for r in result or () if self._filter(r, response, spider)) async def process_spider_output_async(self, response, result, spider): + self._init_depth(response, spider) + async for r in result or (): + if self._filter(r, response, spider): + yield r + + def _init_depth(self, response, spider): # base case (depth=0) if 'depth' not in response.meta: response.meta['depth'] = 0 if self.verbose_stats: self.stats.inc_value('request_depth_count/0', spider=spider) - async for r in result or (): - if self._filter(r, response, spider): - yield r - def _filter(self, request, response, spider): if not isinstance(request, Request): return True