Implement a close reason for robots.txt affecting all start requests

docs/topics/downloader-middleware.rst

@@ -1019,31 +1019,7 @@ RobotsTxtMiddleware
 .. module:: scrapy.downloadermiddlewares.robotstxt
    :synopsis: robots.txt middleware
 
-.. class:: RobotsTxtMiddleware
-
-    This middleware filters out requests forbidden by the robots.txt exclusion
-    standard.
-
-    To make sure Scrapy respects robots.txt make sure the middleware is enabled
-    and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
-
-    The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the
-    user agent string to use for matching in the robots.txt_ file. If it
-    is ``None``, the User-Agent header you are sending with the request or the
-    :setting:`USER_AGENT` setting (in that order) will be used for determining
-    the user agent to use in the robots.txt_ file.
-
-    This middleware has to be combined with a robots.txt_ parser.
-
-    Scrapy ships with support for the following robots.txt_ parsers:
-
-    * :ref:`Protego <protego-parser>` (default)
-    * :ref:`RobotFileParser <python-robotfileparser>`
-    * :ref:`Robotexclusionrulesparser <rerp-parser>`
-    * :ref:`Reppy <reppy-parser>` (deprecated)
-
-    You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
-    setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
+.. autoclass:: RobotsTxtMiddleware
 
 .. reqmeta:: dont_obey_robotstxt
 

scrapy/core/engine.py

@@ -430,6 +430,9 @@ class ExecutionEngine:
 
         -   ``memusage_exceeded``: See
             :class:`~scrapy.extensions.memusage.MemoryUsage`.
+
+        -   ``robotstxt_denied``: See
+            :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
         """
         if self.slot is None:
             raise RuntimeError("Engine slot not assigned")

scrapy/downloadermiddlewares/robotstxt.py

@@ -18,6 +18,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
 from scrapy.robotstxt import RobotParser
+from scrapy.spidermiddlewares.robotstxt import _start_requests_processed
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import load_object
@@ -31,9 +32,44 @@ logger = logging.getLogger(__name__)
 
 
 class RobotsTxtMiddleware:
+    """This middleware filters out requests forbidden by the robots.txt
+    exclusion standard.
+
+    To make sure Scrapy respects robots.txt make sure the middleware is enabled
+    and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
+
+    The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the
+    user agent string to use for matching in the robots.txt_ file. If it
+    is ``None``, the User-Agent header you are sending with the request or the
+    :setting:`USER_AGENT` setting (in that order) will be used for determining
+    the user agent to use in the robots.txt_ file.
+
+    This middleware has to be combined with a robots.txt_ parser.
+
+    Scrapy ships with support for the following robots.txt_ parsers:
+
+    * :ref:`Protego <protego-parser>` (default)
+    * :ref:`RobotFileParser <python-robotfileparser>`
+    * :ref:`Robotexclusionrulesparser <rerp-parser>`
+    * :ref:`Reppy <reppy-parser>` (deprecated)
+
+    You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
+    setting. Or you can also :ref:`implement support for a new parser
+    <support-for-new-robots-parser>`.
+
+    If all start requests from a spider are ignored due to robots.txt rules,
+    the spider close reason becomes ``robotstxt_denied``.
+    """
+
     DOWNLOAD_PRIORITY: int = 1000
 
+    @classmethod
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        return cls(crawler)
+
     def __init__(self, crawler: Crawler):
+        self._forbidden_start_request_count = 0
+        self._total_start_request_count = 0
         if not crawler.settings.getbool("ROBOTSTXT_OBEY"):
             raise NotConfigured
         self._default_useragent: str = crawler.settings.get("USER_AGENT", "Scrapy")
@@ -49,9 +85,13 @@ class RobotsTxtMiddleware:
         # check if parser dependencies are met, this should throw an error otherwise.
         self._parserimpl.from_crawler(self.crawler, b"")
 
-    @classmethod
-    def from_crawler(cls, crawler: Crawler) -> Self:
-        return cls(crawler)
+        crawler.signals.connect(
+            self._start_requests_processed, signal=_start_requests_processed
+        )
+
+    def _start_requests_processed(self, count):
+        self._total_start_request_count = count
+        self._maybe_close()
 
     def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]:
         if request.meta.get("dont_obey_robotstxt"):
@@ -80,6 +120,11 @@ class RobotsTxtMiddleware:
             )
             assert self.crawler.stats
             self.crawler.stats.inc_value("robotstxt/forbidden")
+
+            if request.meta.get("is_start_request", False):
+                self._forbidden_start_request_count += 1
+                self._maybe_close()
+
             raise IgnoreRequest("Forbidden by robots.txt")
 
     def robot_parser(
@@ -148,3 +193,15 @@ class RobotsTxtMiddleware:
         assert isinstance(rp_dfd, Deferred)
         self._parsers[netloc] = None
         rp_dfd.callback(None)
+
+    def _maybe_close(self):
+        if not self._total_start_request_count:
+            return
+        if self._forbidden_start_request_count < self._total_start_request_count:
+            return
+        logger.error(
+            "Stopping the spider, all start requests failed because they "
+            "were rejected based on robots.txt rules. See "
+            "https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#topics-dlmw-robots"
+        )
+        self.crawler.engine.close_spider(self.crawler.spider, "robotstxt_denied")

scrapy/settings/default_settings.py

@@ -305,6 +305,7 @@ SPIDER_MIDDLEWARES_BASE = {
     "scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
     "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
     "scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
+    "scrapy.spidermiddlewares.robotstxt.RobotsTxtSpiderMiddleware": 1000,
     # Spider side
 }
 

scrapy/spidermiddlewares/robotstxt.py

@@ -0,0 +1,20 @@
+_start_requests_processed = object()
+
+
+class RobotsTxtSpiderMiddleware:
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    def __init__(self, crawler):
+        self._send_signal = crawler.signals.send_catch_log
+
+    def process_start_requests(self, start_requests, spider):
+        # Mark start requests and reports to the downloader middleware the
+        # number of them once all have been processed.
+        count = 0
+        for request in start_requests:
+            request.meta["is_start_request"] = True
+            yield request
+            count += 1
+        self._send_signal(_start_requests_processed, count=count)

tests/mockserver.py

@@ -21,6 +21,7 @@ from twisted.web.server import NOT_DONE_YET, GzipEncoderFactory, Site
 from twisted.web.static import File
 from twisted.web.util import redirectTo
 
+from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes, to_unicode
 
 
@@ -271,9 +272,16 @@ class Root(resource.Resource):
 
 
 class MockServer:
+    def __init__(self, resource=None):
+        self._args = []
+        if resource:
+            resource_path = f"{resource.__module__}.{resource.__name__}"
+            self._args.append("--resource")
+            self._args.append(resource_path)
+
     def __enter__(self):
         self.proc = Popen(
-            [sys.executable, "-u", "-m", "tests.mockserver", "-t", "http"],
+            [sys.executable, "-u", "-m", "tests.mockserver", *self._args, "-t", "http"],
             stdout=PIPE,
             env=get_mockserver_env(),
         )
@@ -378,13 +386,14 @@ if __name__ == "__main__":
     parser.add_argument(
         "-t", "--type", type=str, choices=("http", "dns"), default="http"
     )
+    parser.add_argument("--resource", type=str, default="tests.mockserver.Root")
     args = parser.parse_args()
 
     factory: ServerFactory
 
     if args.type == "http":
-        root = Root()
-        factory = Site(root)
+        resource = load_object(args.resource)()
+        factory = Site(resource)
         httpPort = reactor.listenTCP(0, factory)
         contextFactory = ssl_context_factory()
         httpsPort = reactor.listenSSL(0, factory, contextFactory)

tests/test_downloadermiddleware_robotstxt.py

@@ -1,19 +1,38 @@
 from unittest import mock
 
 from twisted.internet import error, reactor
-from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
+from twisted.internet.defer import (
+    Deferred,
+    DeferredList,
+    inlineCallbacks,
+    maybeDeferred,
+)
 from twisted.python import failure
 from twisted.trial import unittest
+from twisted.web.resource import Resource
 
+from scrapy import Spider
 from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
 from scrapy.downloadermiddlewares.robotstxt import logger as mw_module_logger
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
 from scrapy.http.request import NO_CALLBACK
 from scrapy.settings import Settings
+from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer
 from tests.test_robotstxt_interface import reppy_available, rerp_available
 
 
+class RobotsTxtResource(Resource):
+    def getChild(self, name, request):
+        return self
+
+    def render_GET(self, request):
+        if request.path == b"/robots.txt":
+            return b"User-agent: *\n" b"Disallow: /deny/\n"
+        return b"foo"
+
+
 class RobotsTxtMiddlewareTest(unittest.TestCase):
     def setUp(self):
         self.crawler = mock.MagicMock()
@@ -246,6 +265,102 @@ Disallow: /some/randome/page.html
         self.assertEqual(request.url, f"{base_url}/robots.txt")
         self.assertEqual(request.callback, NO_CALLBACK)
 
+    @inlineCallbacks
+    def test_forbidden_start_url(self):
+        class TestSpider(Spider):
+            name = "test"
+
+            def parse(self, response):
+                TestSpider.response = response.text
+
+        settings = {"ROBOTSTXT_OBEY": True}
+        crawler = get_crawler(TestSpider, settings_dict=settings)
+
+        with MockServer(RobotsTxtResource) as server:
+            TestSpider.start_urls = [server.url("/deny/")]
+            yield crawler.crawl()
+
+        self.assertEqual(crawler.stats.get_value("finish_reason"), "robotstxt_denied")
+
+    @inlineCallbacks
+    def test_forbidden_start_urls(self):
+        class TestSpider(Spider):
+            name = "test"
+
+            def parse(self, response):
+                TestSpider.response = response.text
+
+        settings = {"ROBOTSTXT_OBEY": True}
+        crawler = get_crawler(TestSpider, settings_dict=settings)
+
+        with MockServer(RobotsTxtResource) as server:
+            TestSpider.start_urls = [
+                server.url("/deny/foo"),
+                server.url("/deny/bar"),
+                server.url("/deny/baz"),
+            ]
+            yield crawler.crawl()
+
+        self.assertEqual(crawler.stats.get_value("finish_reason"), "robotstxt_denied")
+
+    @inlineCallbacks
+    def test_some_forbidden_start_url(self):
+        class TestSpider(Spider):
+            name = "test"
+
+            def parse(self, response):
+                TestSpider.response = response.text
+
+        settings = {"ROBOTSTXT_OBEY": True}
+        crawler = get_crawler(TestSpider, settings_dict=settings)
+
+        with MockServer(RobotsTxtResource) as server:
+            TestSpider.start_urls = [server.url("/deny"), server.url("/allow")]
+            yield crawler.crawl()
+
+        self.assertEqual(crawler.stats.get_value("finish_reason"), "finished")
+
+    @inlineCallbacks
+    def test_follow_up_forbidden_url(self):
+        settings = {"ROBOTSTXT_OBEY": True}
+        with MockServer(RobotsTxtResource) as server:
+
+            class TestSpider(Spider):
+                name = "test"
+                start_urls = [server.url("/allow/")]
+
+                def parse(self, response):
+                    yield response.follow(server.url("/deny/"))
+
+            crawler = get_crawler(TestSpider, settings_dict=settings)
+            yield crawler.crawl()
+
+        self.assertEqual(crawler.stats.get_value("finish_reason"), "finished")
+
+    @inlineCallbacks
+    def test_forbidden_with_partial_start_request_consumption(self):
+        """With concurrency lower than the number of start requests + 1, the
+        code path followed changes, because ``_total_start_request_count`` is
+        not set in the downloader middleware until *after* some start requests
+        have been processed."""
+        settings = {
+            "CONCURRENT_REQUESTS": 1,
+            "ROBOTSTXT_OBEY": True,
+        }
+        with MockServer(RobotsTxtResource) as server:
+
+            class TestSpider(Spider):
+                name = "test"
+                start_urls = [server.url("/deny/")]
+
+                def parse(self, response):
+                    yield response.follow(server.url("/deny/"))
+
+            crawler = get_crawler(TestSpider, settings_dict=settings)
+            yield crawler.crawl()
+
+        self.assertEqual(crawler.stats.get_value("finish_reason"), "robotstxt_denied")
+
 
 class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
     if not rerp_available():