From 3c5acd252fb27b7ff5a0b6dbcb0b62d9b65526d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 9 Jul 2024 13:09:58 +0200 Subject: [PATCH] Mention scheduled requests as one place not to put responses in --- scrapy/core/downloader/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 60ec957d9..4199ed4e2 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -197,12 +197,14 @@ class Downloader: f"your memory allows it, you may increase " f"RESPONSE_MAX_ACTIVE_SIZE, which should increase your " f"crawl speed. If your code keeps non-weak references to " - f"Response objects, your crawl might get stuck " - f"indefinitely; you can set RESPONSE_MAX_ACTIVE_SIZE to 0 " - f"to disable this limit, but then your code might run out " - f"of memory. This message will only appear the first time " - f"this happens. To learn how often request processing has " - f"been paused during a crawl for this reason, see the " + f"Response objects, e.g. in (scheduled) requests or in a " + f"container within a component, your crawl might get " + f"stuck indefinitely; you can set " + f"RESPONSE_MAX_ACTIVE_SIZE to 0 to disable this limit, " + f"but then your code might run out of memory. This " + f"message will only appear the first time this happens. " + f"To learn how often request processing has been paused " + f"during a crawl for this reason, see the " f"request_backouts/response_max_active_size stat." ) self._count_backout("response_max_active_size")