Merge 5aeced47dc into e28e56aa61

docs/topics/api.rst

@@ -91,14 +91,7 @@ how you :ref:`configure the downloader middlewares
         For an introduction on extensions and a list of available extensions on
         Scrapy see :ref:`topics-extensions`.
 
-    .. attribute:: engine
-
-        The execution engine, which coordinates the core crawling logic
-        between the scheduler, downloader and spiders.
-
-        Some extension may want to access the Scrapy engine, to inspect  or
-        modify the downloader and scheduler behaviour, although this is an
-        advanced use and this API is not yet stable.
+    .. autoattribute:: engine
 
     .. attribute:: spider
 
@@ -269,4 +262,4 @@ Engine API
 ==========
 
 .. autoclass:: scrapy.core.engine.ExecutionEngine()
-   :members: needs_backout
+   :members: close_spider_async, needs_backout

docs/topics/downloader-middleware.rst

@@ -1064,30 +1064,7 @@ RobotsTxtMiddleware
 .. module:: scrapy.downloadermiddlewares.robotstxt
    :synopsis: robots.txt middleware
 
-.. class:: RobotsTxtMiddleware
-
-    This middleware filters out requests forbidden by the robots.txt exclusion
-    standard.
-
-    To make sure Scrapy respects robots.txt make sure the middleware is enabled
-    and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
-
-    The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the
-    user agent string to use for matching in the robots.txt_ file. If it
-    is ``None``, the User-Agent header you are sending with the request or the
-    :setting:`USER_AGENT` setting (in that order) will be used for determining
-    the user agent to use in the robots.txt_ file.
-
-    This middleware has to be combined with a robots.txt_ parser.
-
-    Scrapy ships with support for the following robots.txt_ parsers:
-
-    * :ref:`Protego <protego-parser>` (default)
-    * :ref:`RobotFileParser <python-robotfileparser>`
-    * :ref:`Robotexclusionrulesparser <rerp-parser>`
-
-    You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
-    setting. Or you can also :ref:`implement support for a new parser <support-for-new-robots-parser>`.
+.. autoclass:: RobotsTxtMiddleware()
 
 .. reqmeta:: dont_obey_robotstxt
 

docs/topics/stats.rst

@@ -298,6 +298,10 @@ one per actual value of the placeholder.
 
     -   ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`.
 
+    -   ``robotstxt_denied``: no :ref:`start request <start-requests>` could be
+        crawled, and robots.txt rules denied at least one of them, see
+        :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
+
     -   ``shutdown``: the crawl was interrupted, e.g. by a system signal such
         as ``SIGINT`` (:kbd:`Ctrl-C`).
 

scrapy/core/engine.py

@@ -100,6 +100,15 @@ class _Slot:
 
 
 class ExecutionEngine:
+    """The execution engine manages all the core :ref:`components
+    <topics-components>`, such as the :ref:`scheduler <topics-scheduler>`, the
+    downloader, or the :ref:`spider <topics-spiders>`, at run time.
+
+    Some components access the engine through :attr:`Crawler.engine
+    <scrapy.crawler.Crawler.engine>` to access or modify other components, or
+    use core functionality such as closing the running spider.
+    """
+
     _SLOT_HEARTBEAT_INTERVAL: float = 5.0
 
     def __init__(
@@ -626,9 +635,13 @@ class ExecutionEngine:
         return deferred_from_coro(self.close_spider_async(reason=reason))
 
     async def close_spider_async(self, *, reason: str = "cancelled") -> None:  # noqa: PLR0912
-        """Close (cancel) spider and clear all its outstanding requests.
+        """Stop the crawl with the specified *reason* and clear all its
+        outstanding requests.
 
         .. versionadded:: 2.14
+
+        *reason* is an arbitrary string; see :stat:`finish_reason` for the
+        reasons that built-in components use.
         """
         if self.spider is None:
             raise RuntimeError("Spider not opened")

scrapy/crawler.py

@@ -99,6 +99,7 @@ class _LateAttribute(Generic[_T]):
 
 
 class Crawler:
+    #: Running instance of :class:`~scrapy.core.engine.ExecutionEngine`.
     engine: _LateAttribute[ExecutionEngine] = _LateAttribute()
     extensions: _LateAttribute[ExtensionManager] = _LateAttribute()
     logformatter: _LateAttribute[LogFormatter] = _LateAttribute()

scrapy/downloadermiddlewares/robotstxt.py

@@ -12,7 +12,7 @@ from typing import TYPE_CHECKING
 from twisted.internet.defer import Deferred
 
 from scrapy import signals
-from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.exceptions import CloseSpider, IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
 from scrapy.utils.decorators import _warn_spider_arg
@@ -34,11 +34,43 @@ logger = logging.getLogger(__name__)
 
 
 class RobotsTxtMiddleware:
+    """This middleware filters out requests forbidden by the robots.txt
+    exclusion standard.
+
+    To make sure Scrapy respects robots.txt make sure the middleware is enabled
+    and the :setting:`ROBOTSTXT_OBEY` setting is enabled.
+
+    The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the
+    user agent string to use for matching in the robots.txt_ file. If it
+    is ``None``, the User-Agent header you are sending with the request or the
+    :setting:`USER_AGENT` setting (in that order) will be used for determining
+    the user agent to use in the robots.txt_ file.
+
+    This middleware has to be combined with a robots.txt_ parser.
+
+    Scrapy ships with support for the following robots.txt_ parsers:
+
+    * :ref:`Protego <protego-parser>` (default)
+    * :ref:`RobotFileParser <python-robotfileparser>`
+    * :ref:`Robotexclusionrulesparser <rerp-parser>`
+
+    You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER`
+    setting. Or you can also :ref:`implement support for a new parser
+    <support-for-new-robots-parser>`.
+
+    If no :ref:`start request <start-requests>` can be crawled, and robots.txt
+    rules denied at least one of them, the crawl stops with the
+    ``robotstxt_denied`` :stat:`finish_reason`, as long as
+    :class:`~scrapy.spidermiddlewares.start.StartSpiderMiddleware` is enabled.
+    """
+
     DOWNLOAD_PRIORITY: int = 1000
 
     def __init__(self, crawler: Crawler):
         if not crawler.settings.getbool("ROBOTSTXT_OBEY"):
             raise NotConfigured
+        self._start_request_crawled = False
+        self._start_request_denied = False
         self._default_useragent: str = crawler.settings["USER_AGENT"]
         self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"]
         self.crawler: Crawler = crawler
@@ -51,10 +83,29 @@ class RobotsTxtMiddleware:
         # check if parser dependencies are met, this should throw an error otherwise.
         build_from_crawler(self._parserimpl, self.crawler, b"")
 
+        crawler.signals.connect(
+            self._response_received, signal=signals.response_received
+        )
+        crawler.signals.connect(self._spider_idle, signal=signals.spider_idle)
+
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
+    def _response_received(self, request: Request) -> None:
+        if request.meta.get("is_start_request"):
+            self._start_request_crawled = True
+
+    def _spider_idle(self) -> None:
+        if self._start_request_crawled or not self._start_request_denied:
+            return
+        logger.error(
+            "Stopping the crawl: no start request could be crawled, and at "
+            "least one of them was rejected based on robots.txt rules. See "
+            "https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#topics-dlmw-robots"
+        )
+        raise CloseSpider("robotstxt_denied")
+
     @_warn_spider_arg
     async def process_request(
         self, request: Request, spider: Spider | None = None
@@ -81,6 +132,8 @@ class RobotsTxtMiddleware:
                 extra={"spider": self.crawler.spider},
             )
             self._stats.inc_value("robotstxt/forbidden")
+            if request.meta.get("is_start_request"):
+                self._start_request_denied = True
             raise IgnoreRequest("Forbidden by robots.txt")
 
     async def robot_parser(self, request: Request) -> RobotParser | None:

tests/mockserver/http.py

@@ -73,6 +73,9 @@ class Root(BaseResource):
             b"enc-gb18030",
             Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"),
         )
+        put_child(
+            self, b"robots.txt", Data(b"User-agent: *\nDisallow: /deny\n", "text/plain")
+        )
         put_child(self, b"redirect", Redirect(b"/redirected"))
         put_child(
             self, b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")

tests/test_downloadermiddleware_robotstxt.py

@@ -1,13 +1,14 @@
 from __future__ import annotations
 
 import asyncio
+from typing import TYPE_CHECKING, Any
 from unittest import mock
 
 import pytest
 from twisted.internet.defer import Deferred, DeferredList
 from twisted.python import failure
 
-from scrapy import signals
+from scrapy import Spider, signals
 from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
 from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response, TextResponse
@@ -16,9 +17,15 @@ from scrapy.settings import Settings
 from scrapy.utils.asyncio import call_later
 from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
 from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.test import get_crawler
 from tests.utils.decorators import coroutine_test
 from tests.utils.robotstxt import rerp_available
 
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from tests.mockserver.http import MockServer
+
 
 class TestRobotsTxtMiddleware:
     def setup_method(self) -> None:
@@ -283,6 +290,81 @@ Disallow: /some/randome/page.html
         assert request.callback == NO_CALLBACK
 
 
+class _IgnoreSpider(Spider):
+    name = "test"
+
+    def parse(self, response: Response) -> None:
+        pass
+
+
+class _FollowSpider(Spider):
+    name = "test"
+
+    def parse(self, response: Response) -> Iterable[Request]:
+        yield response.follow("/deny/a")
+
+
+class TestRobotsTxtDeniedCloseReason:
+    @staticmethod
+    async def _finish_reason(
+        spider_cls: type[Spider], settings: dict[str, Any] | None = None, **kwargs: Any
+    ) -> Any:
+        crawler = get_crawler(
+            spider_cls, settings_dict={"ROBOTSTXT_OBEY": True, **(settings or {})}
+        )
+        await crawler.crawl_async(**kwargs)
+        assert crawler.stats
+        return crawler.stats.get_value("finish_reason")
+
+    @coroutine_test
+    async def test_all_denied(self, mockserver: MockServer) -> None:
+        reason = await self._finish_reason(
+            _IgnoreSpider,
+            start_urls=[mockserver.url("/deny/a"), mockserver.url("/deny/b")],
+        )
+        assert reason == "robotstxt_denied"
+
+    @coroutine_test
+    async def test_all_denied_low_concurrency(self, mockserver: MockServer) -> None:
+        reason = await self._finish_reason(
+            _IgnoreSpider,
+            settings={"CONCURRENT_REQUESTS": 1},
+            start_urls=[mockserver.url("/deny/a"), mockserver.url("/deny/b")],
+        )
+        assert reason == "robotstxt_denied"
+
+    @coroutine_test
+    async def test_all_denied_after_redirect(self, mockserver: MockServer) -> None:
+        reason = await self._finish_reason(
+            _IgnoreSpider, start_urls=[mockserver.url("/redirect-to?goto=/deny/a")]
+        )
+        assert reason == "robotstxt_denied"
+
+    @coroutine_test
+    async def test_denied_and_download_failure(self, mockserver: MockServer) -> None:
+        reason = await self._finish_reason(
+            _IgnoreSpider,
+            settings={"RETRY_ENABLED": False},
+            start_urls=[mockserver.url("/deny/a"), mockserver.url("/drop")],
+        )
+        assert reason == "robotstxt_denied"
+
+    @coroutine_test
+    async def test_some_denied(self, mockserver: MockServer) -> None:
+        reason = await self._finish_reason(
+            _IgnoreSpider,
+            start_urls=[mockserver.url("/deny/a"), mockserver.url("/text")],
+        )
+        assert reason == "finished"
+
+    @coroutine_test
+    async def test_denied_follow_up_request(self, mockserver: MockServer) -> None:
+        reason = await self._finish_reason(
+            _FollowSpider, start_urls=[mockserver.url("/text")]
+        )
+        assert reason == "finished"
+
+
 @pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")
 class TestRobotsTxtMiddlewareWithRerp(TestRobotsTxtMiddleware):
     def setup_method(self):