From e121040db0318a6a4059a01626f1c0941dc32b62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Nov 2023 11:54:47 +0100 Subject: [PATCH 1/4] Document built-in spider close reasons --- docs/topics/api.rst | 22 ++++++++++++++-------- scrapy/core/engine.py | 31 ++++++++++++++++++++++++++++++- scrapy/crawler.py | 2 ++ 3 files changed, 46 insertions(+), 9 deletions(-) diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 175c877de..73b747e38 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -81,14 +81,7 @@ how you :ref:`configure the downloader middlewares For an introduction on extensions and a list of available extensions on Scrapy see :ref:`topics-extensions`. - .. attribute:: engine - - The execution engine, which coordinates the core crawling logic - between the scheduler, downloader and spiders. - - Some extension may want to access the Scrapy engine, to inspect or - modify the downloader and scheduler behaviour, although this is an - advanced use and this API is not yet stable. + .. autoattribute:: engine .. attribute:: spider @@ -277,3 +270,16 @@ class (which they all inherit from). Close the given spider. After this is called, no more specific stats can be accessed or collected. + + +.. _engine: + +ExecutionEngine API +=================== + +.. module:: scrapy.core.engine + :synopsis: Execution engine + +.. autoclass:: ExecutionEngine + + .. automethod:: close_spider diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index dd1f56f8c..21393c482 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -82,6 +82,15 @@ class Slot: class ExecutionEngine: + """The execution engine manages all the core :ref:`components + `, such as the :ref:`scheduler `, the + downloader, or the :ref:`spider `, at run time. + + Some components access the engine through :attr:`Crawler.engine + ` to access or modify other components, or + use core functionality such as closing the running spider. + """ + def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None: self.crawler: "Crawler" = crawler self.settings: Settings = crawler.settings @@ -401,7 +410,27 @@ class ExecutionEngine: self.close_spider(self.spider, reason=ex.reason) def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred: - """Close (cancel) spider and clear all its outstanding requests""" + """Stop the crawl with the specified *reason* and clear all its + outstanding requests. + + *reason* is an arbitrary string. Built-in Scrapy :ref:`components + ` use the following reasons: + + - ``finished``: When the crawl finishes normally. + + - ``shutdown``: When stopping the crawl is requested, usually by the + user through a system signal. + + - ``cancelled``: When :exc:`~scrapy.exceptions.CloseSpider` is + raised, e.g. from a spider callback, without a custom *reason*. + + - ``closespider_errorcount``, ``closespider_pagecount``, + ``closespider_itemcount``, ``closespider_timeout_no_item``: See + :class:`~scrapy.extensions.closespider.CloseSpider`. + + - ``memusage_exceeded``: See + :class:`~scrapy.extensions.memusage.MemoryUsage`. + """ if self.slot is None: raise RuntimeError("Engine slot not assigned") diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1d3a11208..afd371e2b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -85,6 +85,8 @@ class Crawler: self.logformatter: Optional[LogFormatter] = None self.request_fingerprinter: Optional[RequestFingerprinter] = None self.spider: Optional[Spider] = None + + #: Running instance of :class:`~scrapy.core.engine.ExecutionEngine`. self.engine: Optional[ExecutionEngine] = None def _update_root_log_handler(self) -> None: From 5409025d1658e38b71015d8e7cbb88728c5edfcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 30 Nov 2023 15:45:08 +0100 Subject: [PATCH 2/4] Implement a close reason for robots.txt affecting all start requests --- docs/topics/downloader-middleware.rst | 26 +---- scrapy/core/engine.py | 3 + scrapy/downloadermiddlewares/robotstxt.py | 63 +++++++++- scrapy/settings/default_settings.py | 1 + scrapy/spidermiddlewares/robotstxt.py | 20 ++++ tests/mockserver.py | 15 ++- tests/test_downloadermiddleware_robotstxt.py | 117 ++++++++++++++++++- 7 files changed, 213 insertions(+), 32 deletions(-) create mode 100644 scrapy/spidermiddlewares/robotstxt.py diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1abbc4968..a567ab6a9 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1019,31 +1019,7 @@ RobotsTxtMiddleware .. module:: scrapy.downloadermiddlewares.robotstxt :synopsis: robots.txt middleware -.. class:: RobotsTxtMiddleware - - This middleware filters out requests forbidden by the robots.txt exclusion - standard. - - To make sure Scrapy respects robots.txt make sure the middleware is enabled - and the :setting:`ROBOTSTXT_OBEY` setting is enabled. - - The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the - user agent string to use for matching in the robots.txt_ file. If it - is ``None``, the User-Agent header you are sending with the request or the - :setting:`USER_AGENT` setting (in that order) will be used for determining - the user agent to use in the robots.txt_ file. - - This middleware has to be combined with a robots.txt_ parser. - - Scrapy ships with support for the following robots.txt_ parsers: - - * :ref:`Protego ` (default) - * :ref:`RobotFileParser ` - * :ref:`Robotexclusionrulesparser ` - * :ref:`Reppy ` (deprecated) - - You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` - setting. Or you can also :ref:`implement support for a new parser `. +.. autoclass:: RobotsTxtMiddleware .. reqmeta:: dont_obey_robotstxt diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 21393c482..18193aff3 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -430,6 +430,9 @@ class ExecutionEngine: - ``memusage_exceeded``: See :class:`~scrapy.extensions.memusage.MemoryUsage`. + + - ``robotstxt_denied``: See + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. """ if self.slot is None: raise RuntimeError("Engine slot not assigned") diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 6a0ecb7bf..2f5a8de36 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -18,6 +18,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.robotstxt import RobotParser +from scrapy.spidermiddlewares.robotstxt import _start_requests_processed from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object @@ -31,9 +32,44 @@ logger = logging.getLogger(__name__) class RobotsTxtMiddleware: + """This middleware filters out requests forbidden by the robots.txt + exclusion standard. + + To make sure Scrapy respects robots.txt make sure the middleware is enabled + and the :setting:`ROBOTSTXT_OBEY` setting is enabled. + + The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the + user agent string to use for matching in the robots.txt_ file. If it + is ``None``, the User-Agent header you are sending with the request or the + :setting:`USER_AGENT` setting (in that order) will be used for determining + the user agent to use in the robots.txt_ file. + + This middleware has to be combined with a robots.txt_ parser. + + Scrapy ships with support for the following robots.txt_ parsers: + + * :ref:`Protego ` (default) + * :ref:`RobotFileParser ` + * :ref:`Robotexclusionrulesparser ` + * :ref:`Reppy ` (deprecated) + + You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` + setting. Or you can also :ref:`implement support for a new parser + `. + + If all start requests from a spider are ignored due to robots.txt rules, + the spider close reason becomes ``robotstxt_denied``. + """ + DOWNLOAD_PRIORITY: int = 1000 + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + def __init__(self, crawler: Crawler): + self._forbidden_start_request_count = 0 + self._total_start_request_count = 0 if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy") @@ -49,9 +85,13 @@ class RobotsTxtMiddleware: # check if parser dependencies are met, this should throw an error otherwise. self._parserimpl.from_crawler(self.crawler, b"") - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler) + crawler.signals.connect( + self._start_requests_processed, signal=_start_requests_processed + ) + + def _start_requests_processed(self, count): + self._total_start_request_count = count + self._maybe_close() def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]: if request.meta.get("dont_obey_robotstxt"): @@ -80,6 +120,11 @@ class RobotsTxtMiddleware: ) assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/forbidden") + + if request.meta.get("is_start_request", False): + self._forbidden_start_request_count += 1 + self._maybe_close() + raise IgnoreRequest("Forbidden by robots.txt") def robot_parser( @@ -148,3 +193,15 @@ class RobotsTxtMiddleware: assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = None rp_dfd.callback(None) + + def _maybe_close(self): + if not self._total_start_request_count: + return + if self._forbidden_start_request_count < self._total_start_request_count: + return + logger.error( + "Stopping the spider, all start requests failed because they " + "were rejected based on robots.txt rules. See " + "https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#topics-dlmw-robots" + ) + self.crawler.engine.close_spider(self.crawler.spider, "robotstxt_denied") diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d6b3585e2..07bc950e0 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -305,6 +305,7 @@ SPIDER_MIDDLEWARES_BASE = { "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, + "scrapy.spidermiddlewares.robotstxt.RobotsTxtSpiderMiddleware": 1000, # Spider side } diff --git a/scrapy/spidermiddlewares/robotstxt.py b/scrapy/spidermiddlewares/robotstxt.py new file mode 100644 index 000000000..3987c7909 --- /dev/null +++ b/scrapy/spidermiddlewares/robotstxt.py @@ -0,0 +1,20 @@ +_start_requests_processed = object() + + +class RobotsTxtSpiderMiddleware: + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + def __init__(self, crawler): + self._send_signal = crawler.signals.send_catch_log + + def process_start_requests(self, start_requests, spider): + # Mark start requests and reports to the downloader middleware the + # number of them once all have been processed. + count = 0 + for request in start_requests: + request.meta["is_start_request"] = True + yield request + count += 1 + self._send_signal(_start_requests_processed, count=count) diff --git a/tests/mockserver.py b/tests/mockserver.py index 647b0682e..60ab92c3b 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -21,6 +21,7 @@ from twisted.web.server import NOT_DONE_YET, GzipEncoderFactory, Site from twisted.web.static import File from twisted.web.util import redirectTo +from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode @@ -271,9 +272,16 @@ class Root(resource.Resource): class MockServer: + def __init__(self, resource=None): + self._args = [] + if resource: + resource_path = f"{resource.__module__}.{resource.__name__}" + self._args.append("--resource") + self._args.append(resource_path) + def __enter__(self): self.proc = Popen( - [sys.executable, "-u", "-m", "tests.mockserver", "-t", "http"], + [sys.executable, "-u", "-m", "tests.mockserver", *self._args, "-t", "http"], stdout=PIPE, env=get_mockserver_env(), ) @@ -378,13 +386,14 @@ if __name__ == "__main__": parser.add_argument( "-t", "--type", type=str, choices=("http", "dns"), default="http" ) + parser.add_argument("--resource", type=str, default="tests.mockserver.Root") args = parser.parse_args() factory: ServerFactory if args.type == "http": - root = Root() - factory = Site(root) + resource = load_object(args.resource)() + factory = Site(resource) httpPort = reactor.listenTCP(0, factory) contextFactory = ssl_context_factory() httpsPort = reactor.listenSSL(0, factory, contextFactory) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 26898a6a1..d29d455ac 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,19 +1,38 @@ from unittest import mock from twisted.internet import error, reactor -from twisted.internet.defer import Deferred, DeferredList, maybeDeferred +from twisted.internet.defer import ( + Deferred, + DeferredList, + inlineCallbacks, + maybeDeferred, +) from twisted.python import failure from twisted.trial import unittest +from twisted.web.resource import Resource +from scrapy import Spider from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware from scrapy.downloadermiddlewares.robotstxt import logger as mw_module_logger from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings +from scrapy.utils.test import get_crawler +from tests.mockserver import MockServer from tests.test_robotstxt_interface import reppy_available, rerp_available +class RobotsTxtResource(Resource): + def getChild(self, name, request): + return self + + def render_GET(self, request): + if request.path == b"/robots.txt": + return b"User-agent: *\n" b"Disallow: /deny/\n" + return b"foo" + + class RobotsTxtMiddlewareTest(unittest.TestCase): def setUp(self): self.crawler = mock.MagicMock() @@ -246,6 +265,102 @@ Disallow: /some/randome/page.html self.assertEqual(request.url, f"{base_url}/robots.txt") self.assertEqual(request.callback, NO_CALLBACK) + @inlineCallbacks + def test_forbidden_start_url(self): + class TestSpider(Spider): + name = "test" + + def parse(self, response): + TestSpider.response = response.text + + settings = {"ROBOTSTXT_OBEY": True} + crawler = get_crawler(TestSpider, settings_dict=settings) + + with MockServer(RobotsTxtResource) as server: + TestSpider.start_urls = [server.url("/deny/")] + yield crawler.crawl() + + self.assertEqual(crawler.stats.get_value("finish_reason"), "robotstxt_denied") + + @inlineCallbacks + def test_forbidden_start_urls(self): + class TestSpider(Spider): + name = "test" + + def parse(self, response): + TestSpider.response = response.text + + settings = {"ROBOTSTXT_OBEY": True} + crawler = get_crawler(TestSpider, settings_dict=settings) + + with MockServer(RobotsTxtResource) as server: + TestSpider.start_urls = [ + server.url("/deny/foo"), + server.url("/deny/bar"), + server.url("/deny/baz"), + ] + yield crawler.crawl() + + self.assertEqual(crawler.stats.get_value("finish_reason"), "robotstxt_denied") + + @inlineCallbacks + def test_some_forbidden_start_url(self): + class TestSpider(Spider): + name = "test" + + def parse(self, response): + TestSpider.response = response.text + + settings = {"ROBOTSTXT_OBEY": True} + crawler = get_crawler(TestSpider, settings_dict=settings) + + with MockServer(RobotsTxtResource) as server: + TestSpider.start_urls = [server.url("/deny"), server.url("/allow")] + yield crawler.crawl() + + self.assertEqual(crawler.stats.get_value("finish_reason"), "finished") + + @inlineCallbacks + def test_follow_up_forbidden_url(self): + settings = {"ROBOTSTXT_OBEY": True} + with MockServer(RobotsTxtResource) as server: + + class TestSpider(Spider): + name = "test" + start_urls = [server.url("/allow/")] + + def parse(self, response): + yield response.follow(server.url("/deny/")) + + crawler = get_crawler(TestSpider, settings_dict=settings) + yield crawler.crawl() + + self.assertEqual(crawler.stats.get_value("finish_reason"), "finished") + + @inlineCallbacks + def test_forbidden_with_partial_start_request_consumption(self): + """With concurrency lower than the number of start requests + 1, the + code path followed changes, because ``_total_start_request_count`` is + not set in the downloader middleware until *after* some start requests + have been processed.""" + settings = { + "CONCURRENT_REQUESTS": 1, + "ROBOTSTXT_OBEY": True, + } + with MockServer(RobotsTxtResource) as server: + + class TestSpider(Spider): + name = "test" + start_urls = [server.url("/deny/")] + + def parse(self, response): + yield response.follow(server.url("/deny/")) + + crawler = get_crawler(TestSpider, settings_dict=settings) + yield crawler.crawl() + + self.assertEqual(crawler.stats.get_value("finish_reason"), "robotstxt_denied") + class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest): if not rerp_available(): From e11a6fb223d01d97475f57c0dea2c1b83bb36810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 14 Dec 2023 11:27:23 +0100 Subject: [PATCH 3/4] New signals: start_request_returned, start_requests_exhausted --- docs/topics/signals.rst | 39 +++++++++++++++++++ scrapy/core/engine.py | 5 +++ scrapy/downloadermiddlewares/robotstxt.py | 46 +++++++++++++++++++---- scrapy/settings/default_settings.py | 1 - scrapy/signals.py | 2 + scrapy/spidermiddlewares/robotstxt.py | 20 ---------- 6 files changed, 84 insertions(+), 29 deletions(-) delete mode 100644 scrapy/spidermiddlewares/robotstxt.py diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 9bfd1761c..36c98cfb7 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -466,6 +466,45 @@ headers_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.Spider` object +start_request_returned +~~~~~~~~~~~~~~~~~~~~~~ + +.. signal:: start_request_returned +.. function:: start_request_returned(request) + + .. versionadded:: VERSION + + Sent after a :class:`~scrapy.http.Request` is returned by the + :meth:`spider.start_requests ` iterator and + processed by the + :meth:`process_start_requests ` + method of :ref:`spider middlewares `, and before + that request reaches the :ref:`scheduler ` + (:signal:`request_scheduled` signal). + + This signal does not support returning deferreds from its handlers. + + :param request: Returned request. + :type request: scrapy.http.Request + +start_requests_exhausted +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. signal:: start_requests_exhausted +.. function:: start_requests_exhausted() + + .. versionadded:: VERSION + + Sent after the :meth:`spider.start_requests ` + iterator (including the :meth:`process_start_requests ` + method of :ref:`spider middlewares `) is + exhausted, either normally or due to an exception. + + The :signal:`start_request_returned` signal will have been called for all + start requests by the time this signal is sent. + + This signal does not support returning deferreds from its handlers. + Response signals ---------------- diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 18193aff3..c8a71907c 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -190,6 +190,7 @@ class ExecutionEngine: request = next(self.slot.start_requests) except StopIteration: self.slot.start_requests = None + self.signals.send_catch_log(signal=signals.start_requests_exhausted) except Exception: self.slot.start_requests = None logger.error( @@ -197,7 +198,11 @@ class ExecutionEngine: exc_info=True, extra={"spider": self.spider}, ) + self.signals.send_catch_log(signal=signals.start_requests_exhausted) else: + self.signals.send_catch_log( + signal=signals.start_request_returned, request=request + ) self.crawl(request) if self.spider_is_idle() and self.slot.close_if_idle: diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 2f5a8de36..1fe419a8b 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,18 +7,17 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, Optional, Set, Union from twisted.internet.defer import Deferred, maybeDeferred from twisted.python.failure import Failure -from scrapy import Spider +from scrapy import Spider, signals from scrapy.crawler import Crawler from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.robotstxt import RobotParser -from scrapy.spidermiddlewares.robotstxt import _start_requests_processed from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import load_object @@ -70,6 +69,8 @@ class RobotsTxtMiddleware: def __init__(self, crawler: Crawler): self._forbidden_start_request_count = 0 self._total_start_request_count = 0 + self._pending_start_request_fingerprints: Set[bytes] = set() + self._exhausted_start_requests = False if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy") @@ -84,27 +85,53 @@ class RobotsTxtMiddleware: # check if parser dependencies are met, this should throw an error otherwise. self._parserimpl.from_crawler(self.crawler, b"") + assert crawler.request_fingerprinter is not None + self._fingerprinter = crawler.request_fingerprinter crawler.signals.connect( - self._start_requests_processed, signal=_start_requests_processed + self._start_request_returned, signal=signals.start_request_returned + ) + crawler.signals.connect( + self._start_requests_exhausted, signal=signals.start_requests_exhausted ) - def _start_requests_processed(self, count): - self._total_start_request_count = count + def _start_request_returned(self, request): + self._total_start_request_count += 1 + fingerprint = self._fingerprinter.fingerprint(request) + self._pending_start_request_fingerprints.add(fingerprint) + + def _start_requests_exhausted(self): + self._exhausted_start_requests = True self._maybe_close() def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]: + fingerprint = self._fingerprinter.fingerprint(request) + if fingerprint in self._pending_start_request_fingerprints: + self._pending_start_request_fingerprints.remove(fingerprint) + is_start_request = True + else: + is_start_request = False + if request.meta.get("dont_obey_robotstxt"): return None if request.url.startswith("data:") or request.url.startswith("file:"): return None d: Deferred = maybeDeferred(self.robot_parser, request, spider) + if is_start_request: + self._pending_start_request_fingerprints.add(fingerprint) d.addCallback(self.process_request_2, request, spider) return d def process_request_2( self, rp: Optional[RobotParser], request: Request, spider: Spider ) -> None: + fingerprint = self._fingerprinter.fingerprint(request) + if fingerprint in self._pending_start_request_fingerprints: + self._pending_start_request_fingerprints.remove(fingerprint) + is_start_request = True + else: + is_start_request = False + if rp is None: return @@ -121,7 +148,7 @@ class RobotsTxtMiddleware: assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/forbidden") - if request.meta.get("is_start_request", False): + if is_start_request: self._forbidden_start_request_count += 1 self._maybe_close() @@ -195,7 +222,10 @@ class RobotsTxtMiddleware: rp_dfd.callback(None) def _maybe_close(self): - if not self._total_start_request_count: + if ( + not self._exhausted_start_requests + or self._pending_start_request_fingerprints + ): return if self._forbidden_start_request_count < self._total_start_request_count: return diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 07bc950e0..d6b3585e2 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -305,7 +305,6 @@ SPIDER_MIDDLEWARES_BASE = { "scrapy.spidermiddlewares.referer.RefererMiddleware": 700, "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800, "scrapy.spidermiddlewares.depth.DepthMiddleware": 900, - "scrapy.spidermiddlewares.robotstxt.RobotsTxtSpiderMiddleware": 1000, # Spider side } diff --git a/scrapy/signals.py b/scrapy/signals.py index 0090f1c8b..16ffbb666 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -24,6 +24,8 @@ item_dropped = object() item_error = object() feed_slot_closed = object() feed_exporter_closed = object() +start_request_returned = object() +start_requests_exhausted = object() # for backward compatibility stats_spider_opened = spider_opened diff --git a/scrapy/spidermiddlewares/robotstxt.py b/scrapy/spidermiddlewares/robotstxt.py deleted file mode 100644 index 3987c7909..000000000 --- a/scrapy/spidermiddlewares/robotstxt.py +++ /dev/null @@ -1,20 +0,0 @@ -_start_requests_processed = object() - - -class RobotsTxtSpiderMiddleware: - @classmethod - def from_crawler(cls, crawler): - return cls(crawler) - - def __init__(self, crawler): - self._send_signal = crawler.signals.send_catch_log - - def process_start_requests(self, start_requests, spider): - # Mark start requests and reports to the downloader middleware the - # number of them once all have been processed. - count = 0 - for request in start_requests: - request.meta["is_start_request"] = True - yield request - count += 1 - self._send_signal(_start_requests_processed, count=count) From cbdab1d56cd14c9758d4376034823a17654aa8f8 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 31 Jul 2026 20:08:03 +0200 Subject: [PATCH 4/4] Modernize --- docs/topics/downloader-middleware.rst | 2 +- docs/topics/stats.rst | 4 ++ scrapy/core/engine.py | 22 +--------- scrapy/downloadermiddlewares/robotstxt.py | 33 ++++++++------ tests/mockserver/http.py | 3 ++ tests/mockserver/robotstxt.py | 29 ------------ tests/test_downloadermiddleware_robotstxt.py | 46 ++++++++++++-------- 7 files changed, 57 insertions(+), 82 deletions(-) delete mode 100644 tests/mockserver/robotstxt.py diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index ac1b0d362..10edab242 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1137,7 +1137,7 @@ RobotsTxtMiddleware .. module:: scrapy.downloadermiddlewares.robotstxt :synopsis: robots.txt middleware -.. autoclass:: RobotsTxtMiddleware +.. autoclass:: RobotsTxtMiddleware() .. reqmeta:: dont_obey_robotstxt diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index c702cefe7..31053ae6e 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -291,6 +291,10 @@ one per actual value of the placeholder. - ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`. + - ``robotstxt_denied``: no :ref:`start request ` could be + crawled, and robots.txt rules denied at least one of them, see + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + - ``shutdown``: the crawl was interrupted, e.g. by a system signal such as ``SIGINT`` (:kbd:`Ctrl-C`). diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index c71983b3d..53903536d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -608,26 +608,8 @@ class ExecutionEngine: .. versionadded:: 2.14 - *reason* is an arbitrary string. Built-in Scrapy :ref:`components - ` use the following reasons: - - - ``finished``: When the crawl finishes normally. - - - ``shutdown``: When stopping the crawl is requested, usually by the - user through a system signal. - - - ``cancelled``: When :exc:`~scrapy.exceptions.CloseSpider` is - raised, e.g. from a spider callback, without a custom *reason*. - - - ``closespider_errorcount``, ``closespider_pagecount``, - ``closespider_itemcount``, ``closespider_timeout_no_item``: See - :class:`~scrapy.extensions.closespider.CloseSpider`. - - - ``memusage_exceeded``: See - :class:`~scrapy.extensions.memusage.MemoryUsage`. - - - ``robotstxt_denied``: See - :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + *reason* is an arbitrary string; see :stat:`finish_reason` for the + reasons that built-in components use. """ if self.spider is None: raise RuntimeError("Spider not opened") diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 8a71b1a24..9b1e1b71f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -57,8 +57,10 @@ class RobotsTxtMiddleware: setting. Or you can also :ref:`implement support for a new parser `. - If all :ref:`start requests ` are ignored due to robots.txt - rules, the spider close reason becomes ``robotstxt_denied``. + If no :ref:`start request ` can be crawled, and robots.txt + rules denied at least one of them, the crawl stops with the + ``robotstxt_denied`` :stat:`finish_reason`, as long as + :class:`~scrapy.spidermiddlewares.start.StartSpiderMiddleware` is enabled. """ DOWNLOAD_PRIORITY: int = 1000 @@ -66,9 +68,8 @@ class RobotsTxtMiddleware: def __init__(self, crawler: Crawler): if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured - self._start_request_count = 0 - self._forbidden_start_request_count = 0 - crawler.signals.connect(self._spider_idle, signal=signals.spider_idle) + self._start_request_crawled = False + self._start_request_denied = False self._default_useragent: str = crawler.settings["USER_AGENT"] self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"] self.crawler: Crawler = crawler @@ -80,19 +81,25 @@ class RobotsTxtMiddleware: # check if parser dependencies are met, this should throw an error otherwise. self._parserimpl.from_crawler(self.crawler, b"") + crawler.signals.connect( + self._response_received, signal=signals.response_received + ) + crawler.signals.connect(self._spider_idle, signal=signals.spider_idle) + @classmethod def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) + def _response_received(self, request: Request) -> None: + if request.meta.get("is_start_request"): + self._start_request_crawled = True + def _spider_idle(self) -> None: - if ( - not self._forbidden_start_request_count - or self._forbidden_start_request_count < self._start_request_count - ): + if self._start_request_crawled or not self._start_request_denied: return logger.error( - "Stopping the spider, all start requests failed because they " - "were rejected based on robots.txt rules. See " + "Stopping the crawl: no start request could be crawled, and at " + "least one of them was rejected based on robots.txt rules. See " "https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#topics-dlmw-robots" ) raise CloseSpider("robotstxt_denied") @@ -101,8 +108,6 @@ class RobotsTxtMiddleware: async def process_request( self, request: Request, spider: Spider | None = None ) -> None: - if request.meta.get("is_start_request"): - self._start_request_count += 1 if request.meta.get("dont_obey_robotstxt"): return if request.url.startswith("data:") or request.url.startswith("file:"): @@ -127,7 +132,7 @@ class RobotsTxtMiddleware: assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/forbidden") if request.meta.get("is_start_request"): - self._forbidden_start_request_count += 1 + self._start_request_denied = True raise IgnoreRequest("Forbidden by robots.txt") async def robot_parser(self, request: Request) -> RobotParser | None: diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py index 7ad873c02..e72057f99 100644 --- a/tests/mockserver/http.py +++ b/tests/mockserver/http.py @@ -64,6 +64,9 @@ class Root(resource.Resource): b"enc-gb18030", Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), ) + self.putChild( + b"robots.txt", Data(b"User-agent: *\nDisallow: /deny\n", "text/plain") + ) self.putChild(b"redirect", Redirect(b"/redirected")) self.putChild( b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") diff --git a/tests/mockserver/robotstxt.py b/tests/mockserver/robotstxt.py deleted file mode 100644 index b3a79fc4b..000000000 --- a/tests/mockserver/robotstxt.py +++ /dev/null @@ -1,29 +0,0 @@ -# This is only used by tests.test_downloadermiddleware_robotstxt - -from __future__ import annotations - -from twisted.web import resource - -from .http_base import BaseMockServer, main_factory - - -class Root(resource.Resource): - def getChild(self, path, request): - return self - - def render_GET(self, request): - if request.path == b"/robots.txt": - return b"User-agent: *\nDisallow: /deny\n" - return b"foo" - - -class RobotsTxtMockServer(BaseMockServer): - listen_https = False - module_name = "tests.mockserver.robotstxt" - - -main = main_factory(Root, listen_https=False) - - -if __name__ == "__main__": - main() diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index ffdec0a70..eb9abe08f 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -17,14 +17,14 @@ from scrapy.settings import Settings from scrapy.utils.asyncio import call_later from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.test import get_crawler -from tests.mockserver.robotstxt import RobotsTxtMockServer from tests.utils.decorators import coroutine_test from tests.utils.robotstxt import rerp_available if TYPE_CHECKING: - from collections.abc import Generator, Iterable + from collections.abc import Iterable from scrapy.crawler import Crawler + from tests.mockserver.http import MockServer class TestRobotsTxtMiddleware: @@ -289,12 +289,6 @@ class _FollowSpider(Spider): class TestRobotsTxtDeniedCloseReason: - @pytest.fixture(scope="class") - @classmethod - def server(cls) -> Generator[RobotsTxtMockServer]: - with RobotsTxtMockServer() as server: - yield server - @staticmethod async def _finish_reason( spider_cls: type[Spider], settings: dict[str, Any] | None = None, **kwargs: Any @@ -307,34 +301,50 @@ class TestRobotsTxtDeniedCloseReason: return crawler.stats.get_value("finish_reason") @coroutine_test - async def test_all_denied(self, server: RobotsTxtMockServer) -> None: + async def test_all_denied(self, mockserver: MockServer) -> None: reason = await self._finish_reason( - _IgnoreSpider, start_urls=[server.url("/deny/a"), server.url("/deny/b")] + _IgnoreSpider, + start_urls=[mockserver.url("/deny/a"), mockserver.url("/deny/b")], ) assert reason == "robotstxt_denied" @coroutine_test - async def test_all_denied_low_concurrency( - self, server: RobotsTxtMockServer - ) -> None: + async def test_all_denied_low_concurrency(self, mockserver: MockServer) -> None: reason = await self._finish_reason( _IgnoreSpider, settings={"CONCURRENT_REQUESTS": 1}, - start_urls=[server.url("/deny/a"), server.url("/deny/b")], + start_urls=[mockserver.url("/deny/a"), mockserver.url("/deny/b")], ) assert reason == "robotstxt_denied" @coroutine_test - async def test_some_denied(self, server: RobotsTxtMockServer) -> None: + async def test_all_denied_after_redirect(self, mockserver: MockServer) -> None: reason = await self._finish_reason( - _IgnoreSpider, start_urls=[server.url("/deny/a"), server.url("/allow")] + _IgnoreSpider, start_urls=[mockserver.url("/redirect-to?goto=/deny/a")] + ) + assert reason == "robotstxt_denied" + + @coroutine_test + async def test_denied_and_download_failure(self, mockserver: MockServer) -> None: + reason = await self._finish_reason( + _IgnoreSpider, + settings={"RETRY_ENABLED": False}, + start_urls=[mockserver.url("/deny/a"), mockserver.url("/drop")], + ) + assert reason == "robotstxt_denied" + + @coroutine_test + async def test_some_denied(self, mockserver: MockServer) -> None: + reason = await self._finish_reason( + _IgnoreSpider, + start_urls=[mockserver.url("/deny/a"), mockserver.url("/text")], ) assert reason == "finished" @coroutine_test - async def test_denied_follow_up_request(self, server: RobotsTxtMockServer) -> None: + async def test_denied_follow_up_request(self, mockserver: MockServer) -> None: reason = await self._finish_reason( - _FollowSpider, start_urls=[server.url("/allow")] + _FollowSpider, start_urls=[mockserver.url("/text")] ) assert reason == "finished"