From 92f2c9e308a5eda361229a8e74f74a21b9ff770a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 2 Feb 2021 15:22:20 +0500
Subject: [PATCH] Move spider middlewares to process_iterable_helper.

---
 scrapy/spidermiddlewares/depth.py     |  4 ++--
 scrapy/spidermiddlewares/offsite.py   | 32 ++++++++++++++-------------
 scrapy/spidermiddlewares/referer.py   |  3 ++-
 scrapy/spidermiddlewares/urllength.py |  3 ++-
 4 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 776a6879a..73079bca9 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -3,10 +3,10 @@ Depth Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
-
 import logging
 
 from scrapy.http import Request
+from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -55,4 +55,4 @@ class DepthMiddleware:
             if self.verbose_stats:
                 self.stats.inc_value('request_depth_count/0', spider=spider)
 
-        return (r for r in result or () if _filter(r))
+        return process_iterable_helper(result or (), in_predicate=_filter)
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 6e4efda97..e7f481269 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -10,6 +10,7 @@ import warnings
 from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -26,21 +27,22 @@ class OffsiteMiddleware:
         return o
 
     def process_spider_output(self, response, result, spider):
-        for x in result:
-            if isinstance(x, Request):
-                if x.dont_filter or self.should_follow(x, spider):
-                    yield x
-                else:
-                    domain = urlparse_cached(x).hostname
-                    if domain and domain not in self.domains_seen:
-                        self.domains_seen.add(domain)
-                        logger.debug(
-                            "Filtered offsite request to %(domain)r: %(request)s",
-                            {'domain': domain, 'request': x}, extra={'spider': spider})
-                        self.stats.inc_value('offsite/domains', spider=spider)
-                    self.stats.inc_value('offsite/filtered', spider=spider)
-            else:
-                yield x
+        def in_predicate(x):
+            if not isinstance(x, Request):
+                return True
+            if x.dont_filter or self.should_follow(x, spider):
+                return True
+            domain = urlparse_cached(x).hostname
+            if domain and domain not in self.domains_seen:
+                self.domains_seen.add(domain)
+                logger.debug(
+                    "Filtered offsite request to %(domain)r: %(request)s",
+                    {'domain': domain, 'request': x}, extra={'spider': spider})
+                self.stats.inc_value('offsite/domains', spider=spider)
+            self.stats.inc_value('offsite/filtered', spider=spider)
+            return False
+
+        return process_iterable_helper(result or (), in_predicate=in_predicate)
 
     def should_follow(self, request, spider):
         regex = self.host_regex
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index f81041376..91c8727e1 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -10,6 +10,7 @@ from w3lib.url import safe_url_string
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
+from scrapy.utils.middlewares import process_iterable_helper
 from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 from scrapy.utils.url import strip_url
@@ -337,7 +338,7 @@ class RefererMiddleware:
                 if referrer is not None:
                     r.headers.setdefault('Referer', referrer)
             return r
-        return (_set_referer(r) for r in result or ())
+        return process_iterable_helper(result or (), processor=_set_referer)
 
     def request_scheduled(self, request, spider):
         # check redirected request to patch "Referer" header if necessary
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index 5be1f80cb..c7359fecd 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -8,6 +8,7 @@ import logging
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
+from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -34,4 +35,4 @@ class UrlLengthMiddleware:
             else:
                 return True
 
-        return (r for r in result or () if _filter(r))
+        return process_iterable_helper(result or (), in_predicate=_filter)