diff --git a/docs/news.rst b/docs/news.rst
index b8b976df9..a3af425ec 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5481,9 +5481,9 @@ New features
* A new ``DNS_RESOLVER`` setting allows enabling IPv6 support
(:issue:`1031`, :issue:`4227`)
-* A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring
- the existing soft limit that pauses request downloads when the total
- response data being processed is too high (:issue:`1410`, :issue:`3551`)
+* A new ``SCRAPER_SLOT_MAX_ACTIVE_SIZE`` setting allows configuring the
+ existing soft limit that pauses request downloads when the total response
+ data being processed is too high (:issue:`1410`, :issue:`3551`)
* A new :setting:`TWISTED_REACTOR` setting allows customizing the
:mod:`~twisted.internet.reactor` that Scrapy uses, allowing to
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index b3c58d6d3..5253c1ed4 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -407,6 +407,46 @@ run:
Because :setting:`SPIDER_MODULES` is a list setting, you can include multiple
modules by separating them with commas.
+.. _crawl-optimization:
+
+Identifying crawl bottlenecks
+=============================
+
+Scrapy exposes ``request_backout_seconds`` stats that show how long request
+scheduling was paused during a crawl, and why:
+
+- ``request_backout_seconds/total``: total time paused for any reason
+- ``request_backout_seconds/concurrency``: time paused because
+ :setting:`CONCURRENT_REQUESTS` was reached
+- ``request_backout_seconds/response_max_active_size``: time paused because
+ :setting:`RESPONSE_MAX_ACTIVE_SIZE` was reached
+
+For example, after a crawl you might see::
+
+ 2025-01-01 00:00:00 [scrapy.statscollectors] INFO: Dumping Scrapy stats:
+ {'request_backout_seconds/concurrency': 12.5,
+ 'request_backout_seconds/response_max_active_size': 45.2,
+ 'request_backout_seconds/total': 57.7,
+ ...}
+
+In this case, the spider spent about 45 seconds paused due to large responses
+in memory. You could:
+
+- Increase :setting:`RESPONSE_MAX_ACTIVE_SIZE` if your machine has enough RAM.
+- Reduce :setting:`CONCURRENT_REQUESTS` to let responses be processed faster.
+- Check that your code doesn't hold strong references to
+ :class:`~scrapy.http.Response` objects longer than necessary.
+
+.. note::
+
+ On Windows, Scrapy uses the `win-precise-time
+ `_ package when available to
+ measure backout durations with higher precision. Install it with:
+
+ .. code-block:: shell
+
+ pip install win-precise-time
+
.. _bans:
Avoiding getting banned
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 700238fe9..83387cd96 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -733,6 +733,7 @@ Those are:
* :reqmeta:`redirect_reasons`
* :reqmeta:`redirect_urls`
* :reqmeta:`referrer_policy`
+* :reqmeta:`response_rough_size`
* :reqmeta:`verbatim_url`
.. reqmeta:: bindaddress
@@ -839,6 +840,20 @@ The meta key is used set retry times per request. When set, the
:reqmeta:`max_retry_times` meta key takes higher precedence over the
:setting:`RETRY_TIMES` setting.
+.. reqmeta:: response_rough_size
+
+response_rough_size
+-------------------
+
+Overrides the :setting:`RESPONSE_ROUGH_SIZE` setting for this specific request.
+
+The value is the estimated size (in bytes) to count toward
+:setting:`RESPONSE_MAX_ACTIVE_SIZE` while this request is being downloaded,
+before its actual response size is known. Set to ``0`` to disable rough-size
+counting for this request.
+
+.. versionadded:: VERSION
+
.. reqmeta:: verbatim_url
verbatim_url
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 455985a56..70ef3f4cf 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1709,6 +1709,71 @@ Adjust redirect request priority relative to original request:
- **a positive priority adjust (default) means higher priority.**
- a negative priority adjust means lower priority.
+.. setting:: RESPONSE_MAX_ACTIVE_SIZE
+
+RESPONSE_MAX_ACTIVE_SIZE
+------------------------
+
+Default: ``5_000_000``
+
+Soft limit (in bytes) on the total size of responses being kept in memory.
+
+This counts both the size of response bodies that have passed through
+:ref:`downloader middlewares ` and remain in
+memory, and the :setting:`rough size ` of requests
+currently being downloaded.
+
+When the total exceeds this value, Scrapy pauses scheduling new requests until
+it drops below the limit.
+
+If you set this to ``0``, the limit is disabled.
+
+Setting this to a lower value reduces memory usage at the cost of crawl speed.
+Setting this to a higher value (or disabling it) improves crawl speed but may
+cause memory issues when responses are large.
+
+When the limit is first reached, Scrapy logs an info-level message explaining
+the situation. Check the ``request_backout_seconds/response_max_active_size``
+stat to see how long request processing has been paused due to this limit over
+the course of a crawl.
+
+.. caution::
+
+ If your code stores strong references to :class:`~scrapy.http.Response`
+ objects (e.g. in a scheduled request's meta or in a component attribute),
+ the garbage collector cannot free them, and the total active size may not
+ drop below the limit. In that case your crawl might get stuck indefinitely.
+ Either avoid storing such references, or set this to ``0`` to disable the
+ limit.
+
+ To check whether your crawl is stuck due to this, connect to the
+ :ref:`telnet console ` and run ``prefs()`` to see
+ the count of live :class:`~scrapy.http.Response` objects. If that count
+ is large and not decreasing, you likely have strong response references.
+ See :ref:`topics-leaks` for details.
+
+.. versionadded:: VERSION
+
+.. setting:: RESPONSE_ROUGH_SIZE
+
+RESPONSE_ROUGH_SIZE
+-------------------
+
+Default: ``1024``
+
+Estimated size (in bytes) to count toward :setting:`RESPONSE_MAX_ACTIVE_SIZE`
+for each request that is currently being downloaded, before its actual response
+size is known.
+
+This allows :setting:`RESPONSE_MAX_ACTIVE_SIZE` to provide backpressure based
+on the number of concurrent in-flight requests, not just already-received
+responses. Once the response arrives, its actual body size is counted instead.
+
+You can override this value on a per-request basis via the
+:reqmeta:`response_rough_size` request meta key.
+
+.. versionadded:: VERSION
+
.. setting:: ROBOTSTXT_OBEY
ROBOTSTXT_OBEY
@@ -1859,19 +1924,6 @@ For available choices, see :setting:`SCHEDULER_MEMORY_QUEUE`.
:start-after: queue-common-starts
:end-before: queue-common-ends
-
-.. setting:: SCRAPER_SLOT_MAX_ACTIVE_SIZE
-
-SCRAPER_SLOT_MAX_ACTIVE_SIZE
-----------------------------
-
-Default: ``5_000_000``
-
-Soft limit (in bytes) for response data being processed.
-
-While the sum of the sizes of all responses being processed is above this value,
-Scrapy does not process new requests.
-
.. setting:: SPIDER_CONTRACTS
SPIDER_CONTRACTS
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 002e58dd4..9c6bc27bd 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -52,7 +52,7 @@ if TYPE_CHECKING:
logger = getLogger(__name__)
-@dataclass(slots=True, eq=False)
+@dataclass(slots=True, eq=False, repr=False)
class Slot:
"""Downloader slot"""
@@ -60,13 +60,20 @@ class Slot:
delay: float
randomize_delay: bool
- active: set[Request] = field(default_factory=set, init=False, repr=False)
+ active: set[Request] = field(default_factory=set, init=False)
queue: deque[tuple[Request, Deferred[Response]]] = field(
- default_factory=deque, init=False, repr=False
+ default_factory=deque, init=False
)
- transferring: set[Request] = field(default_factory=set, init=False, repr=False)
- lastseen: float = field(default=0, init=False, repr=False)
- latercall: CallLaterResult | None = field(default=None, init=False, repr=False)
+ transferring: set[Request] = field(default_factory=set, init=False)
+ lastseen: float = field(default=0, init=False)
+ latercall: CallLaterResult | None = field(default=None, init=False)
+
+ def __repr__(self) -> str:
+ return (
+ f"Slot(concurrency={self.concurrency!r}, "
+ f"delay={self.delay:.2f}, "
+ f"randomize_delay={self.randomize_delay!r})"
+ )
def free_transfer_slots(self) -> int:
return self.concurrency - len(self.transferring)
@@ -165,6 +172,7 @@ class Downloader:
self, request: Request, spider: Spider | None = None
) -> Generator[Deferred[Any], Any, Response | Request]:
self.active.add(request)
+ rough_size = self.middleware._count_rough_size(request)
try:
result: Response | Request = yield (
deferred_from_coro(
@@ -174,6 +182,7 @@ class Downloader:
return result
finally:
self.active.remove(request)
+ self.middleware._discount_rough_size(rough_size)
def _record_backout(self, reason):
last_reason, last_reason_start_time = self._last_backout
@@ -194,15 +203,16 @@ class Downloader:
return True
if (
self._response_max_active_size
- and self.middleware.response_active_size >= self._response_max_active_size
+ and self.middleware.total_active_size >= self._response_max_active_size
):
if not self._response_max_active_size_warned:
self._response_max_active_size_warned = True
logger.info(
f"The active response size, i.e. the total size of all "
f"bodies from responses that have been processed by "
- f"downloader middlewares and remain in memory, is "
- f"{self.middleware.response_active_size} B. The "
+ f"downloader middlewares and remain in memory, plus the "
+ f"rough sizes of in-flight requests, is "
+ f"{self.middleware.total_active_size} B. The "
f"RESPONSE_MAX_ACTIVE_SIZE setting sets its maximum value "
f"at {self._response_max_active_size} B. No more requests "
f"will be processed until active response size lowers. If "
@@ -217,7 +227,7 @@ class Downloader:
f"message will only appear the first time this happens. "
f"To learn how often request processing has been paused "
f"during a crawl for this reason, see the "
- f"request_backouts/response_max_active_size stat."
+ f"request_backout_seconds/response_max_active_size stat."
)
self._record_backout("response_max_active_size")
# Force the garbage collection of response objects. Necessary for
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 2c6e1f278..560456bde 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -39,6 +39,11 @@ class DownloaderMiddlewareManager(MiddlewareManager):
super().__init__(*args, **kwargs)
self.response_active_size = 0
self._tracked_responses = WeakSet()
+ self._rough_active_size = 0
+ assert self.crawler is not None
+ self._response_rough_size: int = self.crawler.settings.getint(
+ "RESPONSE_ROUGH_SIZE"
+ )
@classmethod
def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]:
@@ -57,6 +62,19 @@ class DownloaderMiddlewareManager(MiddlewareManager):
self.methods["process_exception"].appendleft(mw.process_exception)
self._check_mw_method_spider_arg(mw.process_exception)
+ @property
+ def total_active_size(self) -> int:
+ """Sum of sizes of tracked responses and rough sizes of in-flight requests."""
+ return self.response_active_size + self._rough_active_size
+
+ def _count_rough_size(self, request: Request) -> int:
+ size: int = request.meta.get("response_rough_size", self._response_rough_size)
+ self._rough_active_size += size
+ return size
+
+ def _discount_rough_size(self, size: int) -> None:
+ self._rough_active_size -= size
+
def _count_response_size(self, response: Response) -> None:
if response in self._tracked_responses:
return
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 790423d89..93b87917b 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -176,6 +176,8 @@ __all__ = [
"REFERRER_POLICIES",
"REFERRER_POLICY",
"REQUEST_FINGERPRINTER_CLASS",
+ "RESPONSE_MAX_ACTIVE_SIZE",
+ "RESPONSE_ROUGH_SIZE",
"RETRY_ENABLED",
"RETRY_EXCEPTIONS",
"RETRY_GIVE_UP_LOG_LEVEL",
@@ -536,6 +538,7 @@ SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"
SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5_000_000
RESPONSE_MAX_ACTIVE_SIZE = 5_000_000
+RESPONSE_ROUGH_SIZE = 1024
SPIDER_CONTRACTS = {}
SPIDER_CONTRACTS_BASE = {
diff --git a/tests/test_downloader.py b/tests/test_downloader.py
index 15485f6cc..14c49929e 100644
--- a/tests/test_downloader.py
+++ b/tests/test_downloader.py
@@ -15,10 +15,7 @@ from scrapy.utils.test import get_crawler
class SlotTest(unittest.TestCase):
def test_repr(self):
slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)
- assert (
- repr(slot)
- == "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)"
- )
+ assert repr(slot) == "Slot(concurrency=8, delay=0.10, randomize_delay=True)"
class OfflineSpider(Spider):
@@ -151,6 +148,107 @@ class ResponseMaxActiveSizeTest(unittest.TestCase):
)
+class ResponseRoughSizeTest(unittest.TestCase):
+ @pytest.fixture(autouse=True)
+ def use_caplog(self, caplog):
+ self.caplog = caplog
+
+ @deferred_f_from_coro_f
+ async def test_default(self):
+ """A crawl without custom settings has RESPONSE_ROUGH_SIZE set to 1024."""
+ crawler = get_crawler(OfflineSpider)
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ await maybe_deferred_to_future(crawler.crawl())
+ assert crawler.engine.downloader.middleware._response_rough_size == 1024
+
+ @deferred_f_from_coro_f
+ async def test_custom(self):
+ """Setting RESPONSE_ROUGH_SIZE to a custom value changes the rough size."""
+ crawler = get_crawler(OfflineSpider, settings_dict={"RESPONSE_ROUGH_SIZE": 0})
+ with warnings.catch_warnings():
+ warnings.simplefilter("error")
+ await maybe_deferred_to_future(crawler.crawl())
+ assert crawler.engine.downloader.middleware._response_rough_size == 0
+
+ @deferred_f_from_coro_f
+ async def test_rough_size_per_request(self):
+ """response_rough_size meta key overrides RESPONSE_ROUGH_SIZE per request.
+
+ A low RESPONSE_MAX_ACTIVE_SIZE is set so that only requests with the
+ custom rough size of 1 pass; without the override the default 1024 would
+ trigger backout and only one request would be downloaded."""
+
+ class TestSpider(Spider):
+ name = "test"
+ custom_settings = {
+ "RESPONSE_MAX_ACTIVE_SIZE": 512,
+ }
+
+ async def start(self):
+ yield Request("data:,a", meta={"response_rough_size": 1})
+ yield Request("data:,b")
+
+ def parse(self, response):
+ pass
+
+ crawler = get_crawler(TestSpider)
+ self.caplog.clear()
+ with self.caplog.at_level("INFO"):
+ await maybe_deferred_to_future(crawler.crawl())
+
+ active_size_log_count = sum(
+ 1
+ for r in self.caplog.records
+ if str(r.msg).startswith("The active response size")
+ and r.levelname == "INFO"
+ )
+ assert active_size_log_count == 1
+
+ @deferred_f_from_coro_f
+ async def test_rough_size_triggers_backout(self):
+ """Rough sizes of in-flight requests count toward the backpressure limit.
+
+ With RESPONSE_MAX_ACTIVE_SIZE=512 and RESPONSE_ROUGH_SIZE=1024, even a
+ response with an empty body should trigger the backout log."""
+
+ class TestSpider(Spider):
+ name = "test"
+ start_urls = ["data:,", "data:,"]
+ custom_settings = {
+ "RESPONSE_MAX_ACTIVE_SIZE": 512,
+ "RESPONSE_ROUGH_SIZE": 1024,
+ }
+
+ def parse(self, response):
+ pass
+
+ crawler = get_crawler(TestSpider)
+ self.caplog.clear()
+ with self.caplog.at_level("INFO"):
+ await maybe_deferred_to_future(crawler.crawl())
+
+ matching_log_count = 0
+ for log_record in self.caplog.records:
+ if (
+ str(log_record.msg).startswith("The active response size")
+ and log_record.levelname == "INFO"
+ ):
+ matching_log_count += 1
+ assert matching_log_count == 1
+
+ expected_stats = {
+ "request_backout_seconds/response_max_active_size": gt(0),
+ "request_backout_seconds/total": gt(0),
+ }
+ actual_stats = {
+ k: v
+ for k, v in crawler.stats.get_stats().items()
+ if k.startswith("request_backout_seconds/")
+ }
+ assert expected_stats == actual_stats
+
+
class RequestBackoutTest(unittest.TestCase):
@pytest.fixture(autouse=True)
def use_caplog(self, caplog):