diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 9bfd1761c..36c98cfb7 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -466,6 +466,45 @@ headers_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.Spider` object +start_request_returned +~~~~~~~~~~~~~~~~~~~~~~ + +.. signal:: start_request_returned +.. function:: start_request_returned(request) + + .. versionadded:: VERSION + + Sent after a :class:`~scrapy.http.Request` is returned by the + :meth:`spider.start_requests ` iterator and + processed by the + :meth:`process_start_requests ` + method of :ref:`spider middlewares `, and before + that request reaches the :ref:`scheduler ` + (:signal:`request_scheduled` signal). + + This signal does not support returning deferreds from its handlers. + + :param request: Returned request. + :type request: scrapy.http.Request + +start_requests_exhausted +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. signal:: start_requests_exhausted +.. function:: start_requests_exhausted() + + .. versionadded:: VERSION + + Sent after the :meth:`spider.start_requests ` + iterator (including the :meth:`process_start_requests ` + method of :ref:`spider middlewares `) is + exhausted, either normally or due to an exception. + + The :signal:`start_request_returned` signal will have been called for all + start requests by the time this signal is sent. + + This signal does not support returning deferreds from its handlers. + Response signals ---------------- diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 18193aff3..c8a71907c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -190,6 +190,7 @@ class ExecutionEngine: request = next(self.slot.start_requests) except StopIteration: self.slot.start_requests = None + self.signals.send_catch_log(signal=signals.start_requests_exhausted) except Exception: self.slot.start_requests = None logger.error( @@ -197,7 +198,11 @@ class ExecutionEngine: exc_info=True, extra={"spider": self.spider}, ) + self.signals.send_catch_log(signal=signals.start_requests_exhausted) else: + self.signals.send_catch_log( + signal=signals.start_request_returned, request=request + ) self.crawl(request) if self.spider_is_idle() and self.slot.close_if_idle: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 2f5a8de36..1fe419a8b 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,18 +7,17 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Union from twisted.internet.defer import Deferred, maybeDeferred from twisted.python.failure import Failure -from scrapy import Spider +from scrapy import Spider, signals from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.robotstxt import RobotParser -from scrapy.spidermiddlewares.robotstxt import _start_requests_processed from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object @@ -70,6 +69,8 @@ class RobotsTxtMiddleware: def __init__(self, crawler: Crawler): self._forbidden_start_request_count = 0 self._total_start_request_count = 0 + self._pending_start_request_fingerprints: Set[bytes] = set() + self._exhausted_start_requests = False if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy") @@ -84,27 +85,53 @@ class RobotsTxtMiddleware: # check if parser dependencies are met, this should throw an error otherwise. self._parserimpl.from_crawler(self.crawler, b"") + assert crawler.request_fingerprinter is not None + self._fingerprinter = crawler.request_fingerprinter crawler.signals.connect( - self._start_requests_processed, signal=_start_requests_processed + self._start_request_returned, signal=signals.start_request_returned + ) + crawler.signals.connect( + self._start_requests_exhausted, signal=signals.start_requests_exhausted ) - def _start_requests_processed(self, count): - self._total_start_request_count = count + def _start_request_returned(self, request): + self._total_start_request_count += 1 + fingerprint = self._fingerprinter.fingerprint(request) + self._pending_start_request_fingerprints.add(fingerprint) + + def _start_requests_exhausted(self): + self._exhausted_start_requests = True self._maybe_close() def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]: + fingerprint = self._fingerprinter.fingerprint(request) + if fingerprint in self._pending_start_request_fingerprints: + self._pending_start_request_fingerprints.remove(fingerprint) + is_start_request = True + else: + is_start_request = False + if request.meta.get("dont_obey_robotstxt"): return None if request.url.startswith("data:") or request.url.startswith("file:"): return None d: Deferred = maybeDeferred(self.robot_parser, request, spider) + if is_start_request: + self._pending_start_request_fingerprints.add(fingerprint) d.addCallback(self.process_request_2, request, spider) return d def process_request_2( self, rp: Optional[RobotParser], request: Request, spider: Spider ) -> None: + fingerprint = self._fingerprinter.fingerprint(request) + if fingerprint in self._pending_start_request_fingerprints: + self._pending_start_request_fingerprints.remove(fingerprint) + is_start_request = True + else: + is_start_request = False + if rp is None: return @@ -121,7 +148,7 @@ class RobotsTxtMiddleware: assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/forbidden") - if request.meta.get("is_start_request", False): + if is_start_request: self._forbidden_start_request_count += 1 self._maybe_close() @@ -195,7 +222,10 @@ class RobotsTxtMiddleware: rp_dfd.callback(None) def _maybe_close(self): - if not self._total_start_request_count: + if ( + not self._exhausted_start_requests + or self._pending_start_request_fingerprints + ): return if self._forbidden_start_request_count < self._total_start_request_count: return diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 07bc950e0..d6b3585e2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -305,7 +305,6 @@ SPIDER_MIDDLEWARES_BASE = { "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, - "scrapy.spidermiddlewares.robotstxt.RobotsTxtSpiderMiddleware": 1000, # Spider side } diff --git a/scrapy/signals.py b/scrapy/signals.py index 0090f1c8b..16ffbb666 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -24,6 +24,8 @@ item_dropped = object() item_error = object() feed_slot_closed = object() feed_exporter_closed = object() +start_request_returned = object() +start_requests_exhausted = object() # for backward compatibility stats_spider_opened = spider_opened diff --git a/scrapy/spidermiddlewares/robotstxt.py b/scrapy/spidermiddlewares/robotstxt.py deleted file mode 100644 index 3987c7909..000000000 --- a/scrapy/spidermiddlewares/robotstxt.py +++ /dev/null @@ -1,20 +0,0 @@ -_start_requests_processed = object() - - -class RobotsTxtSpiderMiddleware: - @classmethod - def from_crawler(cls, crawler): - return cls(crawler) - - def __init__(self, crawler): - self._send_signal = crawler.signals.send_catch_log - - def process_start_requests(self, start_requests, spider): - # Mark start requests and reports to the downloader middleware the - # number of them once all have been processed. - count = 0 - for request in start_requests: - request.meta["is_start_request"] = True - yield request - count += 1 - self._send_signal(_start_requests_processed, count=count)