RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Modernize

This commit is contained in:
Adrian Chaves 2026-07-31 20:08:03 +02:00
parent d23ab29958
commit cbdab1d56c
7 changed files with 57 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/topics/downloader-middleware.rst
View File

@ -1137,7 +1137,7 @@ RobotsTxtMiddleware
.. module:: scrapy.downloadermiddlewares.robotstxt
:synopsis: robots.txt middleware
.. autoclass:: RobotsTxtMiddleware
.. autoclass:: RobotsTxtMiddleware()
.. reqmeta:: dont_obey_robotstxt

4
docs/topics/stats.rst
View File

@ -291,6 +291,10 @@ one per actual value of the placeholder.
- ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`.
- ``robotstxt_denied``: no :ref:`start request <start-requests>` could be
crawled, and robots.txt rules denied at least one of them, see
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
- ``shutdown``: the crawl was interrupted, e.g. by a system signal such
as ``SIGINT`` (:kbd:`Ctrl-C`).

22
scrapy/core/engine.py
View File

@ -608,26 +608,8 @@ class ExecutionEngine:
.. versionadded:: 2.14
*reason* is an arbitrary string. Built-in Scrapy :ref:`components
<topics-components>` use the following reasons:
- ``finished``: When the crawl finishes normally.
- ``shutdown``: When stopping the crawl is requested, usually by the
user through a system signal.
- ``cancelled``: When :exc:`~scrapy.exceptions.CloseSpider` is
raised, e.g. from a spider callback, without a custom *reason*.
- ``closespider_errorcount``, ``closespider_pagecount``,
``closespider_itemcount``, ``closespider_timeout_no_item``: See
:class:`~scrapy.extensions.closespider.CloseSpider`.
- ``memusage_exceeded``: See
:class:`~scrapy.extensions.memusage.MemoryUsage`.
- ``robotstxt_denied``: See
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
*reason* is an arbitrary string; see :stat:`finish_reason` for the
reasons that built-in components use.
"""
if self.spider is None:
raise RuntimeError("Spider not opened")

33
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -57,8 +57,10 @@ class RobotsTxtMiddleware:
setting. Or you can also :ref:`implement support for a new parser
<support-for-new-robots-parser>`.
If all :ref:`start requests <start-requests>` are ignored due to robots.txt
rules, the spider close reason becomes ``robotstxt_denied``.
If no :ref:`start request <start-requests>` can be crawled, and robots.txt
rules denied at least one of them, the crawl stops with the
``robotstxt_denied`` :stat:`finish_reason`, as long as
:class:`~scrapy.spidermiddlewares.start.StartSpiderMiddleware` is enabled.
"""
DOWNLOAD_PRIORITY: int = 1000
@ -66,9 +68,8 @@ class RobotsTxtMiddleware:
def __init__(self, crawler: Crawler):
if not crawler.settings.getbool("ROBOTSTXT_OBEY"):
raise NotConfigured
self._start_request_count = 0
self._forbidden_start_request_count = 0
crawler.signals.connect(self._spider_idle, signal=signals.spider_idle)
self._start_request_crawled = False
self._start_request_denied = False
self._default_useragent: str = crawler.settings["USER_AGENT"]
self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"]
self.crawler: Crawler = crawler
@ -80,19 +81,25 @@ class RobotsTxtMiddleware:
# check if parser dependencies are met, this should throw an error otherwise.
self._parserimpl.from_crawler(self.crawler, b"")
crawler.signals.connect(
self._response_received, signal=signals.response_received
)
crawler.signals.connect(self._spider_idle, signal=signals.spider_idle)
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler)
def _response_received(self, request: Request) -> None:
if request.meta.get("is_start_request"):
self._start_request_crawled = True
def _spider_idle(self) -> None:
if (
not self._forbidden_start_request_count
or self._forbidden_start_request_count < self._start_request_count
):
if self._start_request_crawled or not self._start_request_denied:
return
logger.error(
"Stopping the spider, all start requests failed because they "
"were rejected based on robots.txt rules. See "
"Stopping the crawl: no start request could be crawled, and at "
"least one of them was rejected based on robots.txt rules. See "
"https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#topics-dlmw-robots"
)
raise CloseSpider("robotstxt_denied")
@ -101,8 +108,6 @@ class RobotsTxtMiddleware:
async def process_request(
self, request: Request, spider: Spider | None = None
) -> None:
if request.meta.get("is_start_request"):
self._start_request_count += 1
if request.meta.get("dont_obey_robotstxt"):
return
if request.url.startswith("data:") or request.url.startswith("file:"):
@ -127,7 +132,7 @@ class RobotsTxtMiddleware:
assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/forbidden")
if request.meta.get("is_start_request"):
self._forbidden_start_request_count += 1
self._start_request_denied = True
raise IgnoreRequest("Forbidden by robots.txt")
async def robot_parser(self, request: Request) -> RobotParser | None:

3
tests/mockserver/http.py
View File

@ -64,6 +64,9 @@ class Root(resource.Resource):
b"enc-gb18030",
Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"),
)
self.putChild(
b"robots.txt", Data(b"User-agent: *\nDisallow: /deny\n", "text/plain")
)
self.putChild(b"redirect", Redirect(b"/redirected"))
self.putChild(
b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")

29
tests/mockserver/robotstxt.py
View File

@ -1,29 +0,0 @@
# This is only used by tests.test_downloadermiddleware_robotstxt
from __future__ import annotations
from twisted.web import resource
from .http_base import BaseMockServer, main_factory
class Root(resource.Resource):
def getChild(self, path, request):
return self
def render_GET(self, request):
if request.path == b"/robots.txt":
return b"User-agent: *\nDisallow: /deny\n"
return b"foo"
class RobotsTxtMockServer(BaseMockServer):
listen_https = False
module_name = "tests.mockserver.robotstxt"
main = main_factory(Root, listen_https=False)
if __name__ == "__main__":
main()

46
tests/test_downloadermiddleware_robotstxt.py
View File

@ -17,14 +17,14 @@ from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.test import get_crawler
from tests.mockserver.robotstxt import RobotsTxtMockServer
from tests.utils.decorators import coroutine_test
from tests.utils.robotstxt import rerp_available
if TYPE_CHECKING:
from collections.abc import Generator, Iterable
from collections.abc import Iterable
from scrapy.crawler import Crawler
from tests.mockserver.http import MockServer
class TestRobotsTxtMiddleware:
@ -289,12 +289,6 @@ class _FollowSpider(Spider):
class TestRobotsTxtDeniedCloseReason:
@pytest.fixture(scope="class")
@classmethod
def server(cls) -> Generator[RobotsTxtMockServer]:
with RobotsTxtMockServer() as server:
yield server
@staticmethod
async def _finish_reason(
spider_cls: type[Spider], settings: dict[str, Any] | None = None, **kwargs: Any
@ -307,34 +301,50 @@ class TestRobotsTxtDeniedCloseReason:
return crawler.stats.get_value("finish_reason")
@coroutine_test
async def test_all_denied(self, server: RobotsTxtMockServer) -> None:
async def test_all_denied(self, mockserver: MockServer) -> None:
reason = await self._finish_reason(
_IgnoreSpider, start_urls=[server.url("/deny/a"), server.url("/deny/b")]
_IgnoreSpider,
start_urls=[mockserver.url("/deny/a"), mockserver.url("/deny/b")],
)
assert reason == "robotstxt_denied"
@coroutine_test
async def test_all_denied_low_concurrency(
self, server: RobotsTxtMockServer
) -> None:
async def test_all_denied_low_concurrency(self, mockserver: MockServer) -> None:
reason = await self._finish_reason(
_IgnoreSpider,
settings={"CONCURRENT_REQUESTS": 1},
start_urls=[server.url("/deny/a"), server.url("/deny/b")],
start_urls=[mockserver.url("/deny/a"), mockserver.url("/deny/b")],
)
assert reason == "robotstxt_denied"
@coroutine_test
async def test_some_denied(self, server: RobotsTxtMockServer) -> None:
async def test_all_denied_after_redirect(self, mockserver: MockServer) -> None:
reason = await self._finish_reason(
_IgnoreSpider, start_urls=[server.url("/deny/a"), server.url("/allow")]
_IgnoreSpider, start_urls=[mockserver.url("/redirect-to?goto=/deny/a")]
)
assert reason == "robotstxt_denied"
@coroutine_test
async def test_denied_and_download_failure(self, mockserver: MockServer) -> None:
reason = await self._finish_reason(
_IgnoreSpider,
settings={"RETRY_ENABLED": False},
start_urls=[mockserver.url("/deny/a"), mockserver.url("/drop")],
)
assert reason == "robotstxt_denied"
@coroutine_test
async def test_some_denied(self, mockserver: MockServer) -> None:
reason = await self._finish_reason(
_IgnoreSpider,
start_urls=[mockserver.url("/deny/a"), mockserver.url("/text")],
)
assert reason == "finished"
@coroutine_test
async def test_denied_follow_up_request(self, server: RobotsTxtMockServer) -> None:
async def test_denied_follow_up_request(self, mockserver: MockServer) -> None:
reason = await self._finish_reason(
_FollowSpider, start_urls=[server.url("/allow")]
_FollowSpider, start_urls=[mockserver.url("/text")]
)
assert reason == "finished"