diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 73079bca9..776a6879a 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -3,10 +3,10 @@ Depth Spider Middleware See documentation in docs/topics/spider-middleware.rst """ + import logging from scrapy.http import Request -from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -55,4 +55,4 @@ class DepthMiddleware: if self.verbose_stats: self.stats.inc_value('request_depth_count/0', spider=spider) - return process_iterable_helper(result or (), in_predicate=_filter) + return (r for r in result or () if _filter(r)) diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index e7f481269..6e4efda97 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -10,7 +10,6 @@ import warnings from scrapy import signals from scrapy.http import Request from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -27,22 +26,21 @@ class OffsiteMiddleware: return o def process_spider_output(self, response, result, spider): - def in_predicate(x): - if not isinstance(x, Request): - return True - if x.dont_filter or self.should_follow(x, spider): - return True - domain = urlparse_cached(x).hostname - if domain and domain not in self.domains_seen: - self.domains_seen.add(domain) - logger.debug( - "Filtered offsite request to %(domain)r: %(request)s", - {'domain': domain, 'request': x}, extra={'spider': spider}) - self.stats.inc_value('offsite/domains', spider=spider) - self.stats.inc_value('offsite/filtered', spider=spider) - return False - - return process_iterable_helper(result or (), in_predicate=in_predicate) + for x in result: + if isinstance(x, Request): + if x.dont_filter or self.should_follow(x, spider): + yield x + else: + domain = urlparse_cached(x).hostname + if domain and domain not in self.domains_seen: + self.domains_seen.add(domain) + logger.debug( + "Filtered offsite request to %(domain)r: %(request)s", + {'domain': domain, 'request': x}, extra={'spider': spider}) + self.stats.inc_value('offsite/domains', spider=spider) + self.stats.inc_value('offsite/filtered', spider=spider) + else: + yield x def should_follow(self, request, spider): regex = self.host_regex diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index 91c8727e1..f81041376 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -10,7 +10,6 @@ from w3lib.url import safe_url_string from scrapy.http import Request, Response from scrapy.exceptions import NotConfigured from scrapy import signals -from scrapy.utils.middlewares import process_iterable_helper from scrapy.utils.python import to_unicode from scrapy.utils.misc import load_object from scrapy.utils.url import strip_url @@ -338,7 +337,7 @@ class RefererMiddleware: if referrer is not None: r.headers.setdefault('Referer', referrer) return r - return process_iterable_helper(result or (), processor=_set_referer) + return (_set_referer(r) for r in result or ()) def request_scheduled(self, request, spider): # check redirected request to patch "Referer" header if necessary diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index c7359fecd..5be1f80cb 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -8,7 +8,6 @@ import logging from scrapy.http import Request from scrapy.exceptions import NotConfigured -from scrapy.utils.middlewares import process_iterable_helper logger = logging.getLogger(__name__) @@ -35,4 +34,4 @@ class UrlLengthMiddleware: else: return True - return process_iterable_helper(result or (), in_predicate=_filter) + return (r for r in result or () if _filter(r))