From 7fdee93e4202a3ee72390719ab86e281d07b0bfd Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Thu, 25 Jun 2026 13:38:55 +0200 Subject: [PATCH] WIP --- docs/conf.py | 1 - docs/topics/signals.rst | 19 ++++++ scrapy/core/downloader/__init__.py | 2 + scrapy/core/engine.py | 72 ++++++++++++++++++-- scrapy/downloadermiddlewares/robotstxt.py | 12 +++- scrapy/extensions/throttle.py | 1 + scrapy/robotstxt.py | 18 +++++ scrapy/settings/default_settings.py | 30 ++++++++ scrapy/signals.py | 1 + tests/test_downloadermiddleware_robotstxt.py | 18 +++++ 10 files changed, 166 insertions(+), 8 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 44de57421..2fc7afe8c 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -145,7 +145,6 @@ coverage_ignore_pyobjects = [ # -- Options for the autodoc extension ---------------------------------------- autodoc_type_aliases = { "BackoffData": "BackoffData", - "GetScopesMethod": "GetScopesMethod", "RequestScopes": "RequestScopes", } diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index d13733623..845d7ab47 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -502,6 +502,25 @@ headers_received :param spider: the spider associated with the response :type spider: :class:`~scrapy.Spider` object +robots_parsed +~~~~~~~~~~~~~ + +.. signal:: robots_parsed +.. function:: robots_parsed(robotparser, request) + + Sent by + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it + downloads and parses a ``robots.txt`` file, for the host that *request* + targets. + + This signal supports :ref:`asynchronous handlers `. + + :param robotparser: the parser holding the parsed ``robots.txt`` contents + :type robotparser: :class:`~scrapy.robotstxt.RobotParser` object + + :param request: the request that triggered the ``robots.txt`` download + :type request: :class:`~scrapy.Request` object + Response signals ---------------- diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 7c0ee0eec..a9778d1d7 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -86,6 +86,8 @@ def _get_concurrency_delay( delay: float = settings.getfloat("DOWNLOAD_DELAY") if hasattr(spider, "download_delay"): delay = spider.download_delay + if settings.get("DOWNLOAD_DELAY_PER_SLOT") is not None: + delay = settings.getfloat("DOWNLOAD_DELAY_PER_SLOT") if hasattr(spider, "max_concurrent_requests"): # pragma: no cover warn_on_deprecated_spider_attribute( diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 1033e874f..9096cff5a 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -129,6 +129,13 @@ class ExecutionEngine: self._start_request_processing_awaitable: ( asyncio.Future[None] | Deferred[None] | None ) = None + # Requests currently held by the throttling manager, waiting for their + # scopes to allow them through to the downloader. + self._throttling_waiting: set[Request] = set() + self._delayed_requests_warn_threshold: int = self.settings.getint( + "DELAYED_REQUESTS_WARN_THRESHOLD" + ) + self._delayed_requests_warned: bool = False downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) try: self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class( @@ -351,6 +358,35 @@ class ExecutionEngine: or bool(self._slot.closing) or self.downloader.needs_backout() or self.scraper.slot.needs_backout() + or self._throttling_needs_backout() + ) + + def _throttling_needs_backout(self) -> bool: + """Apply backpressure while requests are held by the throttling manager. + + Requests waiting on throttling do not occupy a downloader concurrency + slot, so without this check the engine would keep draining the + scheduler into pending throttling waits. Count them together with the + in-flight requests against the global concurrency limit. + """ + if not self._throttling_waiting: + return False + return ( + len(self._throttling_waiting) + len(self.downloader.active) + >= self.downloader.total_concurrency + ) + + def _maybe_warn_delayed_requests(self) -> None: + if self._delayed_requests_warned: + return + if len(self._throttling_waiting) < self._delayed_requests_warn_threshold: + return + self._delayed_requests_warned = True + logger.warning( + f"There are {len(self._throttling_waiting)} requests held back by throttling. This may " + "indicate a throttling configuration that is too aggressive or a " + "site that is rate-limiting heavily. See DELAYED_REQUESTS_WARN_THRESHOLD.", + extra={"spider": self.spider}, ) def _remove_request(self, _: Any, request: Request) -> None: @@ -426,6 +462,8 @@ class ExecutionEngine: return False if self.downloader.active: # downloader has pending requests return False + if self._throttling_waiting: # requests held by the throttling manager + return False if self._start is not None: # not all start requests are handled return False return not self._slot.scheduler.has_pending_requests() @@ -481,6 +519,21 @@ class ExecutionEngine: return await self.download_async(response_or_request) return response_or_request + @inlineCallbacks + def _acquire_throttling( + self, request: Request + ) -> Generator[Deferred[Any], Any, None]: + """Wait at the throttling gate before *request* is sent, tracking it as + held meanwhile.""" + self._throttling_waiting.add(request) + self._maybe_warn_delayed_requests() + throttler = self.crawler.throttler + assert throttler is not None + try: + yield deferred_from_coro(throttler.acquire(request)) + finally: + self._throttling_waiting.discard(request) + @inlineCallbacks def _download( self, request: Request @@ -489,12 +542,19 @@ class ExecutionEngine: assert self.spider is not None self._slot.add_request(request) + throttler = self.crawler.throttler + assert throttler is not None try: - result: Response | Request - if self._downloader_fetch_needs_spider: - result = yield self.downloader.fetch(request, self.spider) - else: - result = yield self.downloader.fetch(request) + yield self._acquire_throttling(request) + try: + result: Response | Request + if self._downloader_fetch_needs_spider: + result = yield self.downloader.fetch(request, self.spider) + else: + result = yield self.downloader.fetch(request) + except Exception as exc: + yield deferred_from_coro(throttler.process_exception(request, exc)) + raise if not isinstance(result, (Response, Request)): raise TypeError( f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}" @@ -502,6 +562,7 @@ class ExecutionEngine: if isinstance(result, Response): if result.request is None: result.request = request + yield deferred_from_coro(throttler.process_response(result)) logkws = self.logformatter.crawled(result.request, result, self.spider) if logkws is not None: logger.log( @@ -515,6 +576,7 @@ class ExecutionEngine: ) return result finally: + throttler.release(request) self._slot.nextcall.schedule() def open_spider( diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 7d0c17884..81a3a887f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred +from scrapy import signals from scrapy.exceptions import IgnoreRequest, NotConfigured from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK @@ -98,7 +99,7 @@ class RobotsTxtMiddleware: assert self.crawler.stats try: resp = await self.crawler.engine.download_async(robotsreq) - self._parse_robots(resp, netloc) + await self._parse_robots(resp, netloc, request) except Exception as e: if not isinstance(e, IgnoreRequest): logger.error( @@ -115,13 +116,20 @@ class RobotsTxtMiddleware: return await maybe_deferred_to_future(parser) return parser - def _parse_robots(self, response: Response, netloc: str) -> None: + async def _parse_robots( + self, response: Response, netloc: str, request: Request + ) -> None: assert self.crawler.stats self.crawler.stats.inc_value("robotstxt/response_count") self.crawler.stats.inc_value( f"robotstxt/response_status_count/{response.status}" ) rp = self._parserimpl.from_crawler(self.crawler, response.body) + await self.crawler.signals.send_catch_log_async( + signal=signals.robots_parsed, + robotparser=rp, + request=request, + ) rp_dfd = self._parsers[netloc] assert isinstance(rp_dfd, Deferred) self._parsers[netloc] = rp diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py index 89fa6ff67..42e58cbc9 100644 --- a/scrapy/extensions/throttle.py +++ b/scrapy/extensions/throttle.py @@ -31,6 +31,7 @@ class AutoThrottle: "backoff settings instead: " "https://docs.scrapy.org/en/latest/topics/throttling.html", ScrapyDeprecationWarning, + stacklevel=2, ) self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG") diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py index 0c64ea5a5..dc3ba9426 100644 --- a/scrapy/robotstxt.py +++ b/scrapy/robotstxt.py @@ -67,6 +67,16 @@ class RobotParser(metaclass=ABCMeta): :type user_agent: str or bytes """ + def crawl_delay(self, user_agent: str | bytes) -> float | None: + """Return the ``Crawl-delay`` directive for ``user_agent`` as a number + of seconds, or ``None`` if it is not set or the backend does not support + it. + + :param user_agent: User agent + :type user_agent: str or bytes + """ + return None + class PythonRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -85,6 +95,10 @@ class PythonRobotParser(RobotParser): url = to_unicode(url) return self.rp.can_fetch(user_agent, url) + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) + class RerpRobotParser(RobotParser): def __init__(self, robotstxt_body: bytes, spider: Spider | None): @@ -121,3 +135,7 @@ class ProtegoRobotParser(RobotParser): user_agent = to_unicode(user_agent) url = to_unicode(url) return self.rp.can_fetch(url, user_agent) + + def crawl_delay(self, user_agent: str | bytes) -> float | None: + delay = self.rp.crawl_delay(to_unicode(user_agent)) + return None if delay is None else float(delay) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 049921ba7..709d55038 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -235,6 +235,19 @@ AWS_SESSION_TOKEN = None AWS_USE_SSL = None AWS_VERIFY = None +BACKOFF_DELAY_FACTOR = 2.0 +BACKOFF_EXCEPTIONS = [ + "scrapy.exceptions.DownloadFailedError", + "scrapy.exceptions.DownloadTimeoutError", + "scrapy.exceptions.ResponseDataLossError", +] +BACKOFF_HTTP_CODES = [429, 502, 503, 504, 520, 521, 522, 523, 524] +BACKOFF_JITTER = 0.1 +BACKOFF_MAX_DELAY = 300.0 +BACKOFF_MIN_DELAY = 1.0 +BACKOFF_WINDOW = 60.0 + + BOT_NAME = "scrapybot" CLOSESPIDER_ERRORCOUNT = 0 @@ -267,6 +280,8 @@ DEFAULT_REQUEST_HEADERS = { "Accept-Language": "en", } +DELAYED_REQUESTS_WARN_THRESHOLD = 500 + DEPTH_LIMIT = 0 DEPTH_PRIORITY = 0 DEPTH_STATS_VERBOSE = False @@ -279,6 +294,7 @@ DNS_TIMEOUT = 60 DOWNLOAD_BIND_ADDRESS = None DOWNLOAD_DELAY = 0 +DOWNLOAD_DELAY_PER_SLOT = None DOWNLOAD_FAIL_ON_DATALOSS = True @@ -488,6 +504,8 @@ PERIODIC_LOG_DELTA = None PERIODIC_LOG_STATS = None PERIODIC_LOG_TIMING_ENABLED = False +RAMPUP_BACKOFF_TARGET = 1 + RANDOMIZE_DOWNLOAD_DELAY = True REACTOR_THREADPOOL_MAXSIZE = 10 @@ -566,6 +584,8 @@ STATS_DUMP = True STATSMAILER_RCPTS = [] +TARGET_RPM = None + TELNETCONSOLE_ENABLED = 1 TELNETCONSOLE_HOST = "127.0.0.1" TELNETCONSOLE_PORT = [6023, 6073] @@ -574,9 +594,19 @@ TELNETCONSOLE_PASSWORD = None TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) +THROTTLING_MANAGER = "scrapy.throttling.ThrottlingManager" +THROTTLING_SCOPE_MANAGER = "scrapy.throttling.ThrottlingScopeManager" +THROTTLING_SCOPES = {} +THROTTLING_WINDOW = 60.0 +THROTTLING_ROBOTSTXT_OBEY = True +THROTTLING_ROBOTSTXT_MAX_DELAY = 60.0 +THROTTLING_SCOPE_MAX_IDLE = 3600.0 +THROTTLING_DEBUG = False + TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver" TWISTED_REACTOR_ENABLED = True + TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" URLLENGTH_LIMIT = 2083 diff --git a/scrapy/signals.py b/scrapy/signals.py index 972f4fd60..3afeb6eab 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -21,6 +21,7 @@ response_received = object() response_downloaded = object() headers_received = object() bytes_received = object() +robots_parsed = object() item_scraped = object() item_dropped = object() item_error = object() diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 082fc743e..061a551e5 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -8,6 +8,7 @@ import pytest from twisted.internet.defer import Deferred, DeferredList from twisted.python import failure +from scrapy import signals from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured from scrapy.http import Request, Response, TextResponse @@ -27,6 +28,7 @@ class TestRobotsTxtMiddleware: self.crawler: mock.MagicMock = mock.MagicMock() self.crawler.settings = Settings() self.crawler.engine.download_async = mock.AsyncMock() + self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[]) def teardown_method(self): del self.crawler @@ -74,6 +76,22 @@ Disallow: /some/randome/page.html Request("http://site.local/wiki/Käyttäjä:"), middleware ) + @coroutine_test + async def test_robotstxt_emits_robots_parsed_signal(self): + crawler = self._get_successful_crawler() + middleware = RobotsTxtMiddleware(crawler) + request = Request("http://site.local/allowed") + await self.assertNotIgnored(request, middleware) + calls = [ + kwargs + for _, kwargs in crawler.signals.send_catch_log_async.call_args_list + if kwargs.get("signal") is signals.robots_parsed + ] + assert len(calls) == 1 + assert calls[0]["request"] is request + assert calls[0]["robotparser"] is not None + assert "crawler" not in calls[0] + @coroutine_test async def test_robotstxt_multiple_reqs(self) -> None: middleware = RobotsTxtMiddleware(self._get_successful_crawler())