From d10fd9224503431930381a53727e32fc0790a865 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Mon, 15 Jun 2026 14:32:37 +0200 Subject: [PATCH] Set a more realistic RESPONSE_ROUGH_SIZE --- docs/topics/settings.rst | 2 +- scrapy/settings/default_settings.py | 2 +- tests/test_downloader.py | 7 +++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 70ef3f4cf..c00cb1d76 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1759,7 +1759,7 @@ the course of a crawl. RESPONSE_ROUGH_SIZE ------------------- -Default: ``1024`` +Default: ``131072`` (128 kiB) Estimated size (in bytes) to count toward :setting:`RESPONSE_MAX_ACTIVE_SIZE` for each request that is currently being downloaded, before its actual response diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 93b87917b..1f6ffc902 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -538,7 +538,7 @@ SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5_000_000 RESPONSE_MAX_ACTIVE_SIZE = 5_000_000 -RESPONSE_ROUGH_SIZE = 1024 +RESPONSE_ROUGH_SIZE = 131072 SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/tests/test_downloader.py b/tests/test_downloader.py index 14c49929e..1469517d5 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -155,12 +155,11 @@ class ResponseRoughSizeTest(unittest.TestCase): @deferred_f_from_coro_f async def test_default(self): - """A crawl without custom settings has RESPONSE_ROUGH_SIZE set to 1024.""" crawler = get_crawler(OfflineSpider) with warnings.catch_warnings(): warnings.simplefilter("error") await maybe_deferred_to_future(crawler.crawl()) - assert crawler.engine.downloader.middleware._response_rough_size == 1024 + assert crawler.engine.downloader.middleware._response_rough_size == 131072 @deferred_f_from_coro_f async def test_custom(self): @@ -176,8 +175,8 @@ class ResponseRoughSizeTest(unittest.TestCase): """response_rough_size meta key overrides RESPONSE_ROUGH_SIZE per request. A low RESPONSE_MAX_ACTIVE_SIZE is set so that only requests with the - custom rough size of 1 pass; without the override the default 1024 would - trigger backout and only one request would be downloaded.""" + custom rough size of 1 pass; without the override the default 131072 + would trigger backout and only one request would be downloaded.""" class TestSpider(Spider): name = "test"