New signals: start_request_returned, start_requests_exhausted

docs/topics/signals.rst

@@ -466,6 +466,45 @@ headers_received
     :param spider: the spider associated with the response
     :type spider: :class:`~scrapy.Spider` object
 
+start_request_returned
+~~~~~~~~~~~~~~~~~~~~~~
+
+.. signal:: start_request_returned
+.. function:: start_request_returned(request)
+
+    .. versionadded:: VERSION
+
+    Sent after a :class:`~scrapy.http.Request` is returned by the
+    :meth:`spider.start_requests <scrapy.Spider.start_requests>` iterator and
+    processed by the
+    :meth:`process_start_requests <scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests>`
+    method of :ref:`spider middlewares <topics-spider-middleware>`, and before
+    that request reaches the :ref:`scheduler <topics-scheduler>`
+    (:signal:`request_scheduled` signal).
+
+    This signal does not support returning deferreds from its handlers.
+
+    :param request: Returned request.
+    :type request: scrapy.http.Request
+
+start_requests_exhausted
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. signal:: start_requests_exhausted
+.. function:: start_requests_exhausted()
+
+    .. versionadded:: VERSION
+
+    Sent after the :meth:`spider.start_requests <scrapy.Spider.start_requests>`
+    iterator (including the :meth:`process_start_requests <scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests>`
+    method of :ref:`spider middlewares <topics-spider-middleware>`) is
+    exhausted, either normally or due to an exception.
+
+    The :signal:`start_request_returned` signal will have been called for all
+    start requests by the time this signal is sent.
+
+    This signal does not support returning deferreds from its handlers.
+
 Response signals
 ----------------
 

scrapy/core/engine.py

@@ -190,6 +190,7 @@ class ExecutionEngine:
                 request = next(self.slot.start_requests)
             except StopIteration:
                 self.slot.start_requests = None
+                self.signals.send_catch_log(signal=signals.start_requests_exhausted)
             except Exception:
                 self.slot.start_requests = None
                 logger.error(
@@ -197,7 +198,11 @@ class ExecutionEngine:
                     exc_info=True,
                     extra={"spider": self.spider},
                 )
+                self.signals.send_catch_log(signal=signals.start_requests_exhausted)
             else:
+                self.signals.send_catch_log(
+                    signal=signals.start_request_returned, request=request
+                )
                 self.crawl(request)
 
         if self.spider_is_idle() and self.slot.close_if_idle:

scrapy/downloadermiddlewares/robotstxt.py

@@ -7,18 +7,17 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Union
 
 from twisted.internet.defer import Deferred, maybeDeferred
 from twisted.python.failure import Failure
 
-from scrapy import Spider
+from scrapy import Spider, signals
 from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
 from scrapy.robotstxt import RobotParser
-from scrapy.spidermiddlewares.robotstxt import _start_requests_processed
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import load_object
@@ -70,6 +69,8 @@ class RobotsTxtMiddleware:
     def __init__(self, crawler: Crawler):
         self._forbidden_start_request_count = 0
         self._total_start_request_count = 0
+        self._pending_start_request_fingerprints: Set[bytes] = set()
+        self._exhausted_start_requests = False
         if not crawler.settings.getbool("ROBOTSTXT_OBEY"):
             raise NotConfigured
         self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy")
@@ -84,27 +85,53 @@ class RobotsTxtMiddleware:
 
         # check if parser dependencies are met, this should throw an error otherwise.
         self._parserimpl.from_crawler(self.crawler, b"")
+        assert crawler.request_fingerprinter is not None
+        self._fingerprinter = crawler.request_fingerprinter
 
         crawler.signals.connect(
-            self._start_requests_processed, signal=_start_requests_processed
+            self._start_request_returned, signal=signals.start_request_returned
+        )
+        crawler.signals.connect(
+            self._start_requests_exhausted, signal=signals.start_requests_exhausted
         )
 
-    def _start_requests_processed(self, count):
-        self._total_start_request_count = count
+    def _start_request_returned(self, request):
+        self._total_start_request_count += 1
+        fingerprint = self._fingerprinter.fingerprint(request)
+        self._pending_start_request_fingerprints.add(fingerprint)
+
+    def _start_requests_exhausted(self):
+        self._exhausted_start_requests = True
         self._maybe_close()
 
     def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]:
+        fingerprint = self._fingerprinter.fingerprint(request)
+        if fingerprint in self._pending_start_request_fingerprints:
+            self._pending_start_request_fingerprints.remove(fingerprint)
+            is_start_request = True
+        else:
+            is_start_request = False
+
         if request.meta.get("dont_obey_robotstxt"):
             return None
         if request.url.startswith("data:") or request.url.startswith("file:"):
             return None
         d: Deferred = maybeDeferred(self.robot_parser, request, spider)
+        if is_start_request:
+            self._pending_start_request_fingerprints.add(fingerprint)
         d.addCallback(self.process_request_2, request, spider)
         return d
 
     def process_request_2(
         self, rp: Optional[RobotParser], request: Request, spider: Spider
     ) -> None:
+        fingerprint = self._fingerprinter.fingerprint(request)
+        if fingerprint in self._pending_start_request_fingerprints:
+            self._pending_start_request_fingerprints.remove(fingerprint)
+            is_start_request = True
+        else:
+            is_start_request = False
+
         if rp is None:
             return
 
@@ -121,7 +148,7 @@ class RobotsTxtMiddleware:
             assert self.crawler.stats
             self.crawler.stats.inc_value("robotstxt/forbidden")
 
-            if request.meta.get("is_start_request", False):
+            if is_start_request:
                 self._forbidden_start_request_count += 1
                 self._maybe_close()
 
@@ -195,7 +222,10 @@ class RobotsTxtMiddleware:
         rp_dfd.callback(None)
 
     def _maybe_close(self):
-        if not self._total_start_request_count:
+        if (
+            not self._exhausted_start_requests
+            or self._pending_start_request_fingerprints
+        ):
             return
         if self._forbidden_start_request_count < self._total_start_request_count:
             return

scrapy/settings/default_settings.py

@@ -305,7 +305,6 @@ SPIDER_MIDDLEWARES_BASE = {
     "scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
     "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
     "scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
-    "scrapy.spidermiddlewares.robotstxt.RobotsTxtSpiderMiddleware": 1000,
     # Spider side
 }
 

scrapy/signals.py

@@ -24,6 +24,8 @@ item_dropped = object()
 item_error = object()
 feed_slot_closed = object()
 feed_exporter_closed = object()
+start_request_returned = object()
+start_requests_exhausted = object()
 
 # for backward compatibility
 stats_spider_opened = spider_opened

scrapy/spidermiddlewares/robotstxt.py

@@ -1,20 +0,0 @@
-_start_requests_processed = object()
-
-
-class RobotsTxtSpiderMiddleware:
-    @classmethod
-    def from_crawler(cls, crawler):
-        return cls(crawler)
-
-    def __init__(self, crawler):
-        self._send_signal = crawler.signals.send_catch_log
-
-    def process_start_requests(self, start_requests, spider):
-        # Mark start requests and reports to the downloader middleware the
-        # number of them once all have been processed.
-        count = 0
-        for request in start_requests:
-            request.meta["is_start_request"] = True
-            yield request
-            count += 1
-        self._send_signal(_start_requests_processed, count=count)