From 537a577268b69c94352b4f54df904f466c6fe1ce Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 15 Jun 2026 14:13:30 +0200 Subject: [PATCH] Complete the implementation --- docs/news.rst | 6 +- docs/topics/practices.rst | 40 ++++++++++ docs/topics/request-response.rst | 15 ++++ docs/topics/settings.rst | 78 ++++++++++++++++---- scrapy/core/downloader/__init__.py | 30 +++++--- scrapy/core/downloader/middleware.py | 18 +++++ scrapy/settings/default_settings.py | 3 + tests/test_downloader.py | 106 ++++++++++++++++++++++++++- 8 files changed, 266 insertions(+), 30 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index b8b976df9..a3af425ec 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5481,9 +5481,9 @@ New features * A new ``DNS_RESOLVER`` setting allows enabling IPv6 support (:issue:`1031`, :issue:`4227`) -* A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring - the existing soft limit that pauses request downloads when the total - response data being processed is too high (:issue:`1410`, :issue:`3551`) +* A new ``SCRAPER_SLOT_MAX_ACTIVE_SIZE`` setting allows configuring the + existing soft limit that pauses request downloads when the total response + data being processed is too high (:issue:`1410`, :issue:`3551`) * A new :setting:`TWISTED_REACTOR` setting allows customizing the :mod:`~twisted.internet.reactor` that Scrapy uses, allowing to diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index b3c58d6d3..5253c1ed4 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -407,6 +407,46 @@ run: Because :setting:`SPIDER_MODULES` is a list setting, you can include multiple modules by separating them with commas. +.. _crawl-optimization: + +Identifying crawl bottlenecks +============================= + +Scrapy exposes ``request_backout_seconds`` stats that show how long request +scheduling was paused during a crawl, and why: + +- ``request_backout_seconds/total``: total time paused for any reason +- ``request_backout_seconds/concurrency``: time paused because + :setting:`CONCURRENT_REQUESTS` was reached +- ``request_backout_seconds/response_max_active_size``: time paused because + :setting:`RESPONSE_MAX_ACTIVE_SIZE` was reached + +For example, after a crawl you might see:: + + 2025-01-01 00:00:00 [scrapy.statscollectors] INFO: Dumping Scrapy stats: + {'request_backout_seconds/concurrency': 12.5, + 'request_backout_seconds/response_max_active_size': 45.2, + 'request_backout_seconds/total': 57.7, + ...} + +In this case, the spider spent about 45 seconds paused due to large responses +in memory. You could: + +- Increase :setting:`RESPONSE_MAX_ACTIVE_SIZE` if your machine has enough RAM. +- Reduce :setting:`CONCURRENT_REQUESTS` to let responses be processed faster. +- Check that your code doesn't hold strong references to + :class:`~scrapy.http.Response` objects longer than necessary. + +.. note:: + + On Windows, Scrapy uses the `win-precise-time + `_ package when available to + measure backout durations with higher precision. Install it with: + + .. code-block:: shell + + pip install win-precise-time + .. _bans: Avoiding getting banned diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 700238fe9..83387cd96 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -733,6 +733,7 @@ Those are: * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`referrer_policy` +* :reqmeta:`response_rough_size` * :reqmeta:`verbatim_url` .. reqmeta:: bindaddress @@ -839,6 +840,20 @@ The meta key is used set retry times per request. When set, the :reqmeta:`max_retry_times` meta key takes higher precedence over the :setting:`RETRY_TIMES` setting. +.. reqmeta:: response_rough_size + +response_rough_size +------------------- + +Overrides the :setting:`RESPONSE_ROUGH_SIZE` setting for this specific request. + +The value is the estimated size (in bytes) to count toward +:setting:`RESPONSE_MAX_ACTIVE_SIZE` while this request is being downloaded, +before its actual response size is known. Set to ``0`` to disable rough-size +counting for this request. + +.. versionadded:: VERSION + .. reqmeta:: verbatim_url verbatim_url diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 455985a56..70ef3f4cf 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1709,6 +1709,71 @@ Adjust redirect request priority relative to original request: - **a positive priority adjust (default) means higher priority.** - a negative priority adjust means lower priority. +.. setting:: RESPONSE_MAX_ACTIVE_SIZE + +RESPONSE_MAX_ACTIVE_SIZE +------------------------ + +Default: ``5_000_000`` + +Soft limit (in bytes) on the total size of responses being kept in memory. + +This counts both the size of response bodies that have passed through +:ref:`downloader middlewares ` and remain in +memory, and the :setting:`rough size ` of requests +currently being downloaded. + +When the total exceeds this value, Scrapy pauses scheduling new requests until +it drops below the limit. + +If you set this to ``0``, the limit is disabled. + +Setting this to a lower value reduces memory usage at the cost of crawl speed. +Setting this to a higher value (or disabling it) improves crawl speed but may +cause memory issues when responses are large. + +When the limit is first reached, Scrapy logs an info-level message explaining +the situation. Check the ``request_backout_seconds/response_max_active_size`` +stat to see how long request processing has been paused due to this limit over +the course of a crawl. + +.. caution:: + + If your code stores strong references to :class:`~scrapy.http.Response` + objects (e.g. in a scheduled request's meta or in a component attribute), + the garbage collector cannot free them, and the total active size may not + drop below the limit. In that case your crawl might get stuck indefinitely. + Either avoid storing such references, or set this to ``0`` to disable the + limit. + + To check whether your crawl is stuck due to this, connect to the + :ref:`telnet console ` and run ``prefs()`` to see + the count of live :class:`~scrapy.http.Response` objects. If that count + is large and not decreasing, you likely have strong response references. + See :ref:`topics-leaks` for details. + +.. versionadded:: VERSION + +.. setting:: RESPONSE_ROUGH_SIZE + +RESPONSE_ROUGH_SIZE +------------------- + +Default: ``1024`` + +Estimated size (in bytes) to count toward :setting:`RESPONSE_MAX_ACTIVE_SIZE` +for each request that is currently being downloaded, before its actual response +size is known. + +This allows :setting:`RESPONSE_MAX_ACTIVE_SIZE` to provide backpressure based +on the number of concurrent in-flight requests, not just already-received +responses. Once the response arrives, its actual body size is counted instead. + +You can override this value on a per-request basis via the +:reqmeta:`response_rough_size` request meta key. + +.. versionadded:: VERSION + .. setting:: ROBOTSTXT_OBEY ROBOTSTXT_OBEY @@ -1859,19 +1924,6 @@ For available choices, see :setting:`SCHEDULER_MEMORY_QUEUE`. :start-after: queue-common-starts :end-before: queue-common-ends - -.. setting:: SCRAPER_SLOT_MAX_ACTIVE_SIZE - -SCRAPER_SLOT_MAX_ACTIVE_SIZE ----------------------------- - -Default: ``5_000_000`` - -Soft limit (in bytes) for response data being processed. - -While the sum of the sizes of all responses being processed is above this value, -Scrapy does not process new requests. - .. setting:: SPIDER_CONTRACTS SPIDER_CONTRACTS diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 002e58dd4..9c6bc27bd 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -52,7 +52,7 @@ if TYPE_CHECKING: logger = getLogger(__name__) -@dataclass(slots=True, eq=False) +@dataclass(slots=True, eq=False, repr=False) class Slot: """Downloader slot""" @@ -60,13 +60,20 @@ class Slot: delay: float randomize_delay: bool - active: set[Request] = field(default_factory=set, init=False, repr=False) + active: set[Request] = field(default_factory=set, init=False) queue: deque[tuple[Request, Deferred[Response]]] = field( - default_factory=deque, init=False, repr=False + default_factory=deque, init=False ) - transferring: set[Request] = field(default_factory=set, init=False, repr=False) - lastseen: float = field(default=0, init=False, repr=False) - latercall: CallLaterResult | None = field(default=None, init=False, repr=False) + transferring: set[Request] = field(default_factory=set, init=False) + lastseen: float = field(default=0, init=False) + latercall: CallLaterResult | None = field(default=None, init=False) + + def __repr__(self) -> str: + return ( + f"Slot(concurrency={self.concurrency!r}, " + f"delay={self.delay:.2f}, " + f"randomize_delay={self.randomize_delay!r})" + ) def free_transfer_slots(self) -> int: return self.concurrency - len(self.transferring) @@ -165,6 +172,7 @@ class Downloader: self, request: Request, spider: Spider | None = None ) -> Generator[Deferred[Any], Any, Response | Request]: self.active.add(request) + rough_size = self.middleware._count_rough_size(request) try: result: Response | Request = yield ( deferred_from_coro( @@ -174,6 +182,7 @@ class Downloader: return result finally: self.active.remove(request) + self.middleware._discount_rough_size(rough_size) def _record_backout(self, reason): last_reason, last_reason_start_time = self._last_backout @@ -194,15 +203,16 @@ class Downloader: return True if ( self._response_max_active_size - and self.middleware.response_active_size >= self._response_max_active_size + and self.middleware.total_active_size >= self._response_max_active_size ): if not self._response_max_active_size_warned: self._response_max_active_size_warned = True logger.info( f"The active response size, i.e. the total size of all " f"bodies from responses that have been processed by " - f"downloader middlewares and remain in memory, is " - f"{self.middleware.response_active_size} B. The " + f"downloader middlewares and remain in memory, plus the " + f"rough sizes of in-flight requests, is " + f"{self.middleware.total_active_size} B. The " f"RESPONSE_MAX_ACTIVE_SIZE setting sets its maximum value " f"at {self._response_max_active_size} B. No more requests " f"will be processed until active response size lowers. If " @@ -217,7 +227,7 @@ class Downloader: f"message will only appear the first time this happens. " f"To learn how often request processing has been paused " f"during a crawl for this reason, see the " - f"request_backouts/response_max_active_size stat." + f"request_backout_seconds/response_max_active_size stat." ) self._record_backout("response_max_active_size") # Force the garbage collection of response objects. Necessary for diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 2c6e1f278..560456bde 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -39,6 +39,11 @@ class DownloaderMiddlewareManager(MiddlewareManager): super().__init__(*args, **kwargs) self.response_active_size = 0 self._tracked_responses = WeakSet() + self._rough_active_size = 0 + assert self.crawler is not None + self._response_rough_size: int = self.crawler.settings.getint( + "RESPONSE_ROUGH_SIZE" + ) @classmethod def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]: @@ -57,6 +62,19 @@ class DownloaderMiddlewareManager(MiddlewareManager): self.methods["process_exception"].appendleft(mw.process_exception) self._check_mw_method_spider_arg(mw.process_exception) + @property + def total_active_size(self) -> int: + """Sum of sizes of tracked responses and rough sizes of in-flight requests.""" + return self.response_active_size + self._rough_active_size + + def _count_rough_size(self, request: Request) -> int: + size: int = request.meta.get("response_rough_size", self._response_rough_size) + self._rough_active_size += size + return size + + def _discount_rough_size(self, size: int) -> None: + self._rough_active_size -= size + def _count_response_size(self, response: Response) -> None: if response in self._tracked_responses: return diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 790423d89..93b87917b 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -176,6 +176,8 @@ __all__ = [ "REFERRER_POLICIES", "REFERRER_POLICY", "REQUEST_FINGERPRINTER_CLASS", + "RESPONSE_MAX_ACTIVE_SIZE", + "RESPONSE_ROUGH_SIZE", "RETRY_ENABLED", "RETRY_EXCEPTIONS", "RETRY_GIVE_UP_LOG_LEVEL", @@ -536,6 +538,7 @@ SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5_000_000 RESPONSE_MAX_ACTIVE_SIZE = 5_000_000 +RESPONSE_ROUGH_SIZE = 1024 SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 15485f6cc..14c49929e 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -15,10 +15,7 @@ from scrapy.utils.test import get_crawler class SlotTest(unittest.TestCase): def test_repr(self): slot = Slot(concurrency=8, delay=0.1, randomize_delay=True) - assert ( - repr(slot) - == "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)" - ) + assert repr(slot) == "Slot(concurrency=8, delay=0.10, randomize_delay=True)" class OfflineSpider(Spider): @@ -151,6 +148,107 @@ class ResponseMaxActiveSizeTest(unittest.TestCase): ) +class ResponseRoughSizeTest(unittest.TestCase): + @pytest.fixture(autouse=True) + def use_caplog(self, caplog): + self.caplog = caplog + + @deferred_f_from_coro_f + async def test_default(self): + """A crawl without custom settings has RESPONSE_ROUGH_SIZE set to 1024.""" + crawler = get_crawler(OfflineSpider) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.engine.downloader.middleware._response_rough_size == 1024 + + @deferred_f_from_coro_f + async def test_custom(self): + """Setting RESPONSE_ROUGH_SIZE to a custom value changes the rough size.""" + crawler = get_crawler(OfflineSpider, settings_dict={"RESPONSE_ROUGH_SIZE": 0}) + with warnings.catch_warnings(): + warnings.simplefilter("error") + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.engine.downloader.middleware._response_rough_size == 0 + + @deferred_f_from_coro_f + async def test_rough_size_per_request(self): + """response_rough_size meta key overrides RESPONSE_ROUGH_SIZE per request. + + A low RESPONSE_MAX_ACTIVE_SIZE is set so that only requests with the + custom rough size of 1 pass; without the override the default 1024 would + trigger backout and only one request would be downloaded.""" + + class TestSpider(Spider): + name = "test" + custom_settings = { + "RESPONSE_MAX_ACTIVE_SIZE": 512, + } + + async def start(self): + yield Request("data:,a", meta={"response_rough_size": 1}) + yield Request("data:,b") + + def parse(self, response): + pass + + crawler = get_crawler(TestSpider) + self.caplog.clear() + with self.caplog.at_level("INFO"): + await maybe_deferred_to_future(crawler.crawl()) + + active_size_log_count = sum( + 1 + for r in self.caplog.records + if str(r.msg).startswith("The active response size") + and r.levelname == "INFO" + ) + assert active_size_log_count == 1 + + @deferred_f_from_coro_f + async def test_rough_size_triggers_backout(self): + """Rough sizes of in-flight requests count toward the backpressure limit. + + With RESPONSE_MAX_ACTIVE_SIZE=512 and RESPONSE_ROUGH_SIZE=1024, even a + response with an empty body should trigger the backout log.""" + + class TestSpider(Spider): + name = "test" + start_urls = ["data:,", "data:,"] + custom_settings = { + "RESPONSE_MAX_ACTIVE_SIZE": 512, + "RESPONSE_ROUGH_SIZE": 1024, + } + + def parse(self, response): + pass + + crawler = get_crawler(TestSpider) + self.caplog.clear() + with self.caplog.at_level("INFO"): + await maybe_deferred_to_future(crawler.crawl()) + + matching_log_count = 0 + for log_record in self.caplog.records: + if ( + str(log_record.msg).startswith("The active response size") + and log_record.levelname == "INFO" + ): + matching_log_count += 1 + assert matching_log_count == 1 + + expected_stats = { + "request_backout_seconds/response_max_active_size": gt(0), + "request_backout_seconds/total": gt(0), + } + actual_stats = { + k: v + for k, v in crawler.stats.get_stats().items() + if k.startswith("request_backout_seconds/") + } + assert expected_stats == actual_stats + + class RequestBackoutTest(unittest.TestCase): @pytest.fixture(autouse=True) def use_caplog(self, caplog):