diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 598edfeb5..8ac52e082 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -91,14 +91,7 @@ how you :ref:`configure the downloader middlewares For an introduction on extensions and a list of available extensions on Scrapy see :ref:`topics-extensions`. - .. attribute:: engine - - The execution engine, which coordinates the core crawling logic - between the scheduler, downloader and spiders. - - Some extension may want to access the Scrapy engine, to inspect or - modify the downloader and scheduler behaviour, although this is an - advanced use and this API is not yet stable. + .. autoattribute:: engine .. attribute:: spider @@ -269,4 +262,4 @@ Engine API ========== .. autoclass:: scrapy.core.engine.ExecutionEngine() - :members: needs_backout + :members: close_spider_async, needs_backout diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index dec9904d2..72661a52c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1064,30 +1064,7 @@ RobotsTxtMiddleware .. module:: scrapy.downloadermiddlewares.robotstxt :synopsis: robots.txt middleware -.. class:: RobotsTxtMiddleware - - This middleware filters out requests forbidden by the robots.txt exclusion - standard. - - To make sure Scrapy respects robots.txt make sure the middleware is enabled - and the :setting:`ROBOTSTXT_OBEY` setting is enabled. - - The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the - user agent string to use for matching in the robots.txt_ file. If it - is ``None``, the User-Agent header you are sending with the request or the - :setting:`USER_AGENT` setting (in that order) will be used for determining - the user agent to use in the robots.txt_ file. - - This middleware has to be combined with a robots.txt_ parser. - - Scrapy ships with support for the following robots.txt_ parsers: - - * :ref:`Protego ` (default) - * :ref:`RobotFileParser ` - * :ref:`Robotexclusionrulesparser ` - - You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` - setting. Or you can also :ref:`implement support for a new parser `. +.. autoclass:: RobotsTxtMiddleware() .. reqmeta:: dont_obey_robotstxt diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index 6fb7783d1..d96bf431e 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -298,6 +298,10 @@ one per actual value of the placeholder. - ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`. + - ``robotstxt_denied``: no :ref:`start request ` could be + crawled, and robots.txt rules denied at least one of them, see + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + - ``shutdown``: the crawl was interrupted, e.g. by a system signal such as ``SIGINT`` (:kbd:`Ctrl-C`). diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 104daf399..318a36a22 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -100,6 +100,15 @@ class _Slot: class ExecutionEngine: + """The execution engine manages all the core :ref:`components + `, such as the :ref:`scheduler `, the + downloader, or the :ref:`spider `, at run time. + + Some components access the engine through :attr:`Crawler.engine + ` to access or modify other components, or + use core functionality such as closing the running spider. + """ + _SLOT_HEARTBEAT_INTERVAL: float = 5.0 def __init__( @@ -626,9 +635,13 @@ class ExecutionEngine: return deferred_from_coro(self.close_spider_async(reason=reason)) async def close_spider_async(self, *, reason: str = "cancelled") -> None: # noqa: PLR0912 - """Close (cancel) spider and clear all its outstanding requests. + """Stop the crawl with the specified *reason* and clear all its + outstanding requests. .. versionadded:: 2.14 + + *reason* is an arbitrary string; see :stat:`finish_reason` for the + reasons that built-in components use. """ if self.spider is None: raise RuntimeError("Spider not opened") diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 444a5fb67..2804da593 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -99,6 +99,7 @@ class _LateAttribute(Generic[_T]): class Crawler: + #: Running instance of :class:`~scrapy.core.engine.ExecutionEngine`. engine: _LateAttribute[ExecutionEngine] = _LateAttribute() extensions: _LateAttribute[ExtensionManager] = _LateAttribute() logformatter: _LateAttribute[LogFormatter] = _LateAttribute() diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 016cf9acb..e1bcb71d7 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred from scrapy import signals -from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.exceptions import CloseSpider, IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.utils.decorators import _warn_spider_arg @@ -34,11 +34,43 @@ logger = logging.getLogger(__name__) class RobotsTxtMiddleware: + """This middleware filters out requests forbidden by the robots.txt + exclusion standard. + + To make sure Scrapy respects robots.txt make sure the middleware is enabled + and the :setting:`ROBOTSTXT_OBEY` setting is enabled. + + The :setting:`ROBOTSTXT_USER_AGENT` setting can be used to specify the + user agent string to use for matching in the robots.txt_ file. If it + is ``None``, the User-Agent header you are sending with the request or the + :setting:`USER_AGENT` setting (in that order) will be used for determining + the user agent to use in the robots.txt_ file. + + This middleware has to be combined with a robots.txt_ parser. + + Scrapy ships with support for the following robots.txt_ parsers: + + * :ref:`Protego ` (default) + * :ref:`RobotFileParser ` + * :ref:`Robotexclusionrulesparser ` + + You can change the robots.txt_ parser with the :setting:`ROBOTSTXT_PARSER` + setting. Or you can also :ref:`implement support for a new parser + `. + + If no :ref:`start request ` can be crawled, and robots.txt + rules denied at least one of them, the crawl stops with the + ``robotstxt_denied`` :stat:`finish_reason`, as long as + :class:`~scrapy.spidermiddlewares.start.StartSpiderMiddleware` is enabled. + """ + DOWNLOAD_PRIORITY: int = 1000 def __init__(self, crawler: Crawler): if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured + self._start_request_crawled = False + self._start_request_denied = False self._default_useragent: str = crawler.settings["USER_AGENT"] self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"] self.crawler: Crawler = crawler @@ -51,10 +83,29 @@ class RobotsTxtMiddleware: # check if parser dependencies are met, this should throw an error otherwise. build_from_crawler(self._parserimpl, self.crawler, b"") + crawler.signals.connect( + self._response_received, signal=signals.response_received + ) + crawler.signals.connect(self._spider_idle, signal=signals.spider_idle) + @classmethod def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) + def _response_received(self, request: Request) -> None: + if request.meta.get("is_start_request"): + self._start_request_crawled = True + + def _spider_idle(self) -> None: + if self._start_request_crawled or not self._start_request_denied: + return + logger.error( + "Stopping the crawl: no start request could be crawled, and at " + "least one of them was rejected based on robots.txt rules. See " + "https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#topics-dlmw-robots" + ) + raise CloseSpider("robotstxt_denied") + @_warn_spider_arg async def process_request( self, request: Request, spider: Spider | None = None @@ -81,6 +132,8 @@ class RobotsTxtMiddleware: extra={"spider": self.crawler.spider}, ) self._stats.inc_value("robotstxt/forbidden") + if request.meta.get("is_start_request"): + self._start_request_denied = True raise IgnoreRequest("Forbidden by robots.txt") async def robot_parser(self, request: Request) -> RobotParser | None: diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py index 6074f1475..5226b8f5b 100644 --- a/tests/mockserver/http.py +++ b/tests/mockserver/http.py @@ -73,6 +73,9 @@ class Root(BaseResource): b"enc-gb18030", Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), ) + put_child( + self, b"robots.txt", Data(b"User-agent: *\nDisallow: /deny\n", "text/plain") + ) put_child(self, b"redirect", Redirect(b"/redirected")) put_child( self, b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 5c9a5af7b..0f508b5fb 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -1,13 +1,14 @@ from __future__ import annotations import asyncio +from typing import TYPE_CHECKING, Any from unittest import mock import pytest from twisted.internet.defer import Deferred, DeferredList from twisted.python import failure -from scrapy import signals +from scrapy import Spider, signals from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse @@ -16,9 +17,15 @@ from scrapy.settings import Settings from scrapy.utils.asyncio import call_later from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.misc import build_from_crawler +from scrapy.utils.test import get_crawler from tests.utils.decorators import coroutine_test from tests.utils.robotstxt import rerp_available +if TYPE_CHECKING: + from collections.abc import Iterable + + from tests.mockserver.http import MockServer + class TestRobotsTxtMiddleware: def setup_method(self) -> None: @@ -283,6 +290,81 @@ Disallow: /some/randome/page.html assert request.callback == NO_CALLBACK +class _IgnoreSpider(Spider): + name = "test" + + def parse(self, response: Response) -> None: + pass + + +class _FollowSpider(Spider): + name = "test" + + def parse(self, response: Response) -> Iterable[Request]: + yield response.follow("/deny/a") + + +class TestRobotsTxtDeniedCloseReason: + @staticmethod + async def _finish_reason( + spider_cls: type[Spider], settings: dict[str, Any] | None = None, **kwargs: Any + ) -> Any: + crawler = get_crawler( + spider_cls, settings_dict={"ROBOTSTXT_OBEY": True, **(settings or {})} + ) + await crawler.crawl_async(**kwargs) + assert crawler.stats + return crawler.stats.get_value("finish_reason") + + @coroutine_test + async def test_all_denied(self, mockserver: MockServer) -> None: + reason = await self._finish_reason( + _IgnoreSpider, + start_urls=[mockserver.url("/deny/a"), mockserver.url("/deny/b")], + ) + assert reason == "robotstxt_denied" + + @coroutine_test + async def test_all_denied_low_concurrency(self, mockserver: MockServer) -> None: + reason = await self._finish_reason( + _IgnoreSpider, + settings={"CONCURRENT_REQUESTS": 1}, + start_urls=[mockserver.url("/deny/a"), mockserver.url("/deny/b")], + ) + assert reason == "robotstxt_denied" + + @coroutine_test + async def test_all_denied_after_redirect(self, mockserver: MockServer) -> None: + reason = await self._finish_reason( + _IgnoreSpider, start_urls=[mockserver.url("/redirect-to?goto=/deny/a")] + ) + assert reason == "robotstxt_denied" + + @coroutine_test + async def test_denied_and_download_failure(self, mockserver: MockServer) -> None: + reason = await self._finish_reason( + _IgnoreSpider, + settings={"RETRY_ENABLED": False}, + start_urls=[mockserver.url("/deny/a"), mockserver.url("/drop")], + ) + assert reason == "robotstxt_denied" + + @coroutine_test + async def test_some_denied(self, mockserver: MockServer) -> None: + reason = await self._finish_reason( + _IgnoreSpider, + start_urls=[mockserver.url("/deny/a"), mockserver.url("/text")], + ) + assert reason == "finished" + + @coroutine_test + async def test_denied_follow_up_request(self, mockserver: MockServer) -> None: + reason = await self._finish_reason( + _FollowSpider, start_urls=[mockserver.url("/text")] + ) + assert reason == "finished" + + @pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed") class TestRobotsTxtMiddlewareWithRerp(TestRobotsTxtMiddleware): def setup_method(self):