From cbdab1d56cd14c9758d4376034823a17654aa8f8 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 31 Jul 2026 20:08:03 +0200 Subject: [PATCH] Modernize --- docs/topics/downloader-middleware.rst | 2 +- docs/topics/stats.rst | 4 ++ scrapy/core/engine.py | 22 +--------- scrapy/downloadermiddlewares/robotstxt.py | 33 ++++++++------ tests/mockserver/http.py | 3 ++ tests/mockserver/robotstxt.py | 29 ------------ tests/test_downloadermiddleware_robotstxt.py | 46 ++++++++++++-------- 7 files changed, 57 insertions(+), 82 deletions(-) delete mode 100644 tests/mockserver/robotstxt.py diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index ac1b0d362..10edab242 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -1137,7 +1137,7 @@ RobotsTxtMiddleware .. module:: scrapy.downloadermiddlewares.robotstxt :synopsis: robots.txt middleware -.. autoclass:: RobotsTxtMiddleware +.. autoclass:: RobotsTxtMiddleware() .. reqmeta:: dont_obey_robotstxt diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst index c702cefe7..31053ae6e 100644 --- a/docs/topics/stats.rst +++ b/docs/topics/stats.rst @@ -291,6 +291,10 @@ one per actual value of the placeholder. - ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`. + - ``robotstxt_denied``: no :ref:`start request ` could be + crawled, and robots.txt rules denied at least one of them, see + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + - ``shutdown``: the crawl was interrupted, e.g. by a system signal such as ``SIGINT`` (:kbd:`Ctrl-C`). diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index c71983b3d..53903536d 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -608,26 +608,8 @@ class ExecutionEngine: .. versionadded:: 2.14 - *reason* is an arbitrary string. Built-in Scrapy :ref:`components - ` use the following reasons: - - - ``finished``: When the crawl finishes normally. - - - ``shutdown``: When stopping the crawl is requested, usually by the - user through a system signal. - - - ``cancelled``: When :exc:`~scrapy.exceptions.CloseSpider` is - raised, e.g. from a spider callback, without a custom *reason*. - - - ``closespider_errorcount``, ``closespider_pagecount``, - ``closespider_itemcount``, ``closespider_timeout_no_item``: See - :class:`~scrapy.extensions.closespider.CloseSpider`. - - - ``memusage_exceeded``: See - :class:`~scrapy.extensions.memusage.MemoryUsage`. - - - ``robotstxt_denied``: See - :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`. + *reason* is an arbitrary string; see :stat:`finish_reason` for the + reasons that built-in components use. """ if self.spider is None: raise RuntimeError("Spider not opened") diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 8a71b1a24..9b1e1b71f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -57,8 +57,10 @@ class RobotsTxtMiddleware: setting. Or you can also :ref:`implement support for a new parser `. - If all :ref:`start requests ` are ignored due to robots.txt - rules, the spider close reason becomes ``robotstxt_denied``. + If no :ref:`start request ` can be crawled, and robots.txt + rules denied at least one of them, the crawl stops with the + ``robotstxt_denied`` :stat:`finish_reason`, as long as + :class:`~scrapy.spidermiddlewares.start.StartSpiderMiddleware` is enabled. """ DOWNLOAD_PRIORITY: int = 1000 @@ -66,9 +68,8 @@ class RobotsTxtMiddleware: def __init__(self, crawler: Crawler): if not crawler.settings.getbool("ROBOTSTXT_OBEY"): raise NotConfigured - self._start_request_count = 0 - self._forbidden_start_request_count = 0 - crawler.signals.connect(self._spider_idle, signal=signals.spider_idle) + self._start_request_crawled = False + self._start_request_denied = False self._default_useragent: str = crawler.settings["USER_AGENT"] self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"] self.crawler: Crawler = crawler @@ -80,19 +81,25 @@ class RobotsTxtMiddleware: # check if parser dependencies are met, this should throw an error otherwise. self._parserimpl.from_crawler(self.crawler, b"") + crawler.signals.connect( + self._response_received, signal=signals.response_received + ) + crawler.signals.connect(self._spider_idle, signal=signals.spider_idle) + @classmethod def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) + def _response_received(self, request: Request) -> None: + if request.meta.get("is_start_request"): + self._start_request_crawled = True + def _spider_idle(self) -> None: - if ( - not self._forbidden_start_request_count - or self._forbidden_start_request_count < self._start_request_count - ): + if self._start_request_crawled or not self._start_request_denied: return logger.error( - "Stopping the spider, all start requests failed because they " - "were rejected based on robots.txt rules. See " + "Stopping the crawl: no start request could be crawled, and at " + "least one of them was rejected based on robots.txt rules. See " "https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#topics-dlmw-robots" ) raise CloseSpider("robotstxt_denied") @@ -101,8 +108,6 @@ class RobotsTxtMiddleware: async def process_request( self, request: Request, spider: Spider | None = None ) -> None: - if request.meta.get("is_start_request"): - self._start_request_count += 1 if request.meta.get("dont_obey_robotstxt"): return if request.url.startswith("data:") or request.url.startswith("file:"): @@ -127,7 +132,7 @@ class RobotsTxtMiddleware: assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/forbidden") if request.meta.get("is_start_request"): - self._forbidden_start_request_count += 1 + self._start_request_denied = True raise IgnoreRequest("Forbidden by robots.txt") async def robot_parser(self, request: Request) -> RobotParser | None: diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py index 7ad873c02..e72057f99 100644 --- a/tests/mockserver/http.py +++ b/tests/mockserver/http.py @@ -64,6 +64,9 @@ class Root(resource.Resource): b"enc-gb18030", Data(b"

gb18030 encoding

", "text/html; charset=gb18030"), ) + self.putChild( + b"robots.txt", Data(b"User-agent: *\nDisallow: /deny\n", "text/plain") + ) self.putChild(b"redirect", Redirect(b"/redirected")) self.putChild( b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected") diff --git a/tests/mockserver/robotstxt.py b/tests/mockserver/robotstxt.py deleted file mode 100644 index b3a79fc4b..000000000 --- a/tests/mockserver/robotstxt.py +++ /dev/null @@ -1,29 +0,0 @@ -# This is only used by tests.test_downloadermiddleware_robotstxt - -from __future__ import annotations - -from twisted.web import resource - -from .http_base import BaseMockServer, main_factory - - -class Root(resource.Resource): - def getChild(self, path, request): - return self - - def render_GET(self, request): - if request.path == b"/robots.txt": - return b"User-agent: *\nDisallow: /deny\n" - return b"foo" - - -class RobotsTxtMockServer(BaseMockServer): - listen_https = False - module_name = "tests.mockserver.robotstxt" - - -main = main_factory(Root, listen_https=False) - - -if __name__ == "__main__": - main() diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index ffdec0a70..eb9abe08f 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -17,14 +17,14 @@ from scrapy.settings import Settings from scrapy.utils.asyncio import call_later from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.test import get_crawler -from tests.mockserver.robotstxt import RobotsTxtMockServer from tests.utils.decorators import coroutine_test from tests.utils.robotstxt import rerp_available if TYPE_CHECKING: - from collections.abc import Generator, Iterable + from collections.abc import Iterable from scrapy.crawler import Crawler + from tests.mockserver.http import MockServer class TestRobotsTxtMiddleware: @@ -289,12 +289,6 @@ class _FollowSpider(Spider): class TestRobotsTxtDeniedCloseReason: - @pytest.fixture(scope="class") - @classmethod - def server(cls) -> Generator[RobotsTxtMockServer]: - with RobotsTxtMockServer() as server: - yield server - @staticmethod async def _finish_reason( spider_cls: type[Spider], settings: dict[str, Any] | None = None, **kwargs: Any @@ -307,34 +301,50 @@ class TestRobotsTxtDeniedCloseReason: return crawler.stats.get_value("finish_reason") @coroutine_test - async def test_all_denied(self, server: RobotsTxtMockServer) -> None: + async def test_all_denied(self, mockserver: MockServer) -> None: reason = await self._finish_reason( - _IgnoreSpider, start_urls=[server.url("/deny/a"), server.url("/deny/b")] + _IgnoreSpider, + start_urls=[mockserver.url("/deny/a"), mockserver.url("/deny/b")], ) assert reason == "robotstxt_denied" @coroutine_test - async def test_all_denied_low_concurrency( - self, server: RobotsTxtMockServer - ) -> None: + async def test_all_denied_low_concurrency(self, mockserver: MockServer) -> None: reason = await self._finish_reason( _IgnoreSpider, settings={"CONCURRENT_REQUESTS": 1}, - start_urls=[server.url("/deny/a"), server.url("/deny/b")], + start_urls=[mockserver.url("/deny/a"), mockserver.url("/deny/b")], ) assert reason == "robotstxt_denied" @coroutine_test - async def test_some_denied(self, server: RobotsTxtMockServer) -> None: + async def test_all_denied_after_redirect(self, mockserver: MockServer) -> None: reason = await self._finish_reason( - _IgnoreSpider, start_urls=[server.url("/deny/a"), server.url("/allow")] + _IgnoreSpider, start_urls=[mockserver.url("/redirect-to?goto=/deny/a")] + ) + assert reason == "robotstxt_denied" + + @coroutine_test + async def test_denied_and_download_failure(self, mockserver: MockServer) -> None: + reason = await self._finish_reason( + _IgnoreSpider, + settings={"RETRY_ENABLED": False}, + start_urls=[mockserver.url("/deny/a"), mockserver.url("/drop")], + ) + assert reason == "robotstxt_denied" + + @coroutine_test + async def test_some_denied(self, mockserver: MockServer) -> None: + reason = await self._finish_reason( + _IgnoreSpider, + start_urls=[mockserver.url("/deny/a"), mockserver.url("/text")], ) assert reason == "finished" @coroutine_test - async def test_denied_follow_up_request(self, server: RobotsTxtMockServer) -> None: + async def test_denied_follow_up_request(self, mockserver: MockServer) -> None: reason = await self._finish_reason( - _FollowSpider, start_urls=[server.url("/allow")] + _FollowSpider, start_urls=[mockserver.url("/text")] ) assert reason == "finished"