From 8c5fa6e6aea333e1647f16dd7133e9db4f550a65 Mon Sep 17 00:00:00 2001 From: Vasiliy Kiryanov Date: Mon, 27 Oct 2025 10:42:43 -0400 Subject: [PATCH] Line buffering (#7094) --- scrapy/dupefilters.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 2e264ade3..f0a6988c8 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -73,7 +73,11 @@ class RFPDupeFilter(BaseDupeFilter): self.debug = debug self.logger = logging.getLogger(__name__) if path: - self.file = Path(path, "requests.seen").open("a+", encoding="utf-8") + # line-by-line writing, see: https://github.com/scrapy/scrapy/issues/6019 + self.file = Path(path, "requests.seen").open( + "a+", buffering=1, encoding="utf-8" + ) + self.file.reconfigure(write_through=True) self.file.seek(0) self.fingerprints.update(x.rstrip() for x in self.file)