From f9a5385146c741825d8c0b089cf8e8318dca9403 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 19 Feb 2021 21:19:21 +0500
Subject: [PATCH] Revert "Move spider middlewares to process_iterable_helper."

This reverts commit 92f2c9e308a5eda361229a8e74f74a21b9ff770a.
---
 scrapy/spidermiddlewares/depth.py     |  4 ++--
 scrapy/spidermiddlewares/offsite.py   | 32 +++++++++++++--------------
 scrapy/spidermiddlewares/referer.py   |  3 +--
 scrapy/spidermiddlewares/urllength.py |  3 +--
 4 files changed, 19 insertions(+), 23 deletions(-)

diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 73079bca9..776a6879a 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -3,10 +3,10 @@ Depth Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
+
 import logging
 
 from scrapy.http import Request
-from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -55,4 +55,4 @@ class DepthMiddleware:
             if self.verbose_stats:
                 self.stats.inc_value('request_depth_count/0', spider=spider)
 
-        return process_iterable_helper(result or (), in_predicate=_filter)
+        return (r for r in result or () if _filter(r))
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index e7f481269..6e4efda97 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -10,7 +10,6 @@ import warnings
 from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -27,22 +26,21 @@ class OffsiteMiddleware:
         return o
 
     def process_spider_output(self, response, result, spider):
-        def in_predicate(x):
-            if not isinstance(x, Request):
-                return True
-            if x.dont_filter or self.should_follow(x, spider):
-                return True
-            domain = urlparse_cached(x).hostname
-            if domain and domain not in self.domains_seen:
-                self.domains_seen.add(domain)
-                logger.debug(
-                    "Filtered offsite request to %(domain)r: %(request)s",
-                    {'domain': domain, 'request': x}, extra={'spider': spider})
-                self.stats.inc_value('offsite/domains', spider=spider)
-            self.stats.inc_value('offsite/filtered', spider=spider)
-            return False
-
-        return process_iterable_helper(result or (), in_predicate=in_predicate)
+        for x in result:
+            if isinstance(x, Request):
+                if x.dont_filter or self.should_follow(x, spider):
+                    yield x
+                else:
+                    domain = urlparse_cached(x).hostname
+                    if domain and domain not in self.domains_seen:
+                        self.domains_seen.add(domain)
+                        logger.debug(
+                            "Filtered offsite request to %(domain)r: %(request)s",
+                            {'domain': domain, 'request': x}, extra={'spider': spider})
+                        self.stats.inc_value('offsite/domains', spider=spider)
+                    self.stats.inc_value('offsite/filtered', spider=spider)
+            else:
+                yield x
 
     def should_follow(self, request, spider):
         regex = self.host_regex
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 91c8727e1..f81041376 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -10,7 +10,6 @@ from w3lib.url import safe_url_string
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
-from scrapy.utils.middlewares import process_iterable_helper
 from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 from scrapy.utils.url import strip_url
@@ -338,7 +337,7 @@ class RefererMiddleware:
                 if referrer is not None:
                     r.headers.setdefault('Referer', referrer)
             return r
-        return process_iterable_helper(result or (), processor=_set_referer)
+        return (_set_referer(r) for r in result or ())
 
     def request_scheduled(self, request, spider):
         # check redirected request to patch "Referer" header if necessary
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index c7359fecd..5be1f80cb 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -8,7 +8,6 @@ import logging
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
-from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -35,4 +34,4 @@ class UrlLengthMiddleware:
             else:
                 return True
 
-        return process_iterable_helper(result or (), in_predicate=_filter)
+        return (r for r in result or () if _filter(r))