From ec7dbfd6a589a35d9536f1c2c1890bd36d182015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Tue, 9 Jul 2024 13:01:52 +0200 Subject: [PATCH] Force garbage collection for PyPy support --- scrapy/core/downloader/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index eac68e071..60ec957d9 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,5 +1,6 @@ from __future__ import annotations +import gc import random import warnings from collections import deque @@ -205,6 +206,9 @@ class Downloader: f"request_backouts/response_max_active_size stat." ) self._count_backout("response_max_active_size") + # Force the garbage collection of response objects. Necessary for + # PyPy, which is lazier when it comes to garbage collection. + gc.collect() return True return False