RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extract more common code.

This commit is contained in:
Andrey Rakhmatullin 2022-07-28 13:44:36 +05:00
parent 0f1112f3e2
commit c7b90c6e1e
1 changed files with 7 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
scrapy/spidermiddlewares/depth.py
View File

@ -28,25 +28,22 @@ class DepthMiddleware:
return cls(maxdepth, crawler.stats, verbose, prio)
def process_spider_output(self, response, result, spider):
# base case (depth=0)
if 'depth' not in response.meta:
response.meta['depth'] = 0
if self.verbose_stats:
self.stats.inc_value('request_depth_count/0', spider=spider)
self._init_depth(response, spider)
return (r for r in result or () if self._filter(r, response, spider))
async def process_spider_output_async(self, response, result, spider):
self._init_depth(response, spider)
async for r in result or ():
if self._filter(r, response, spider):
yield r
def _init_depth(self, response, spider):
# base case (depth=0)
if 'depth' not in response.meta:
response.meta['depth'] = 0
if self.verbose_stats:
self.stats.inc_value('request_depth_count/0', spider=spider)
async for r in result or ():
if self._filter(r, response, spider):
yield r
def _filter(self, request, response, spider):
if not isinstance(request, Request):
return True