RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Line buffering (#7094)

This commit is contained in:
Vasiliy Kiryanov 2025-10-27 10:42:43 -04:00 committed by GitHub
parent 804ae167df
commit 8c5fa6e6ae
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/dupefilters.py
View File

@ -73,7 +73,11 @@ class RFPDupeFilter(BaseDupeFilter):
self.debug = debug
self.logger = logging.getLogger(__name__)
if path:
self.file = Path(path, "requests.seen").open("a+", encoding="utf-8")
# line-by-line writing, see: https://github.com/scrapy/scrapy/issues/6019
self.file = Path(path, "requests.seen").open(
"a+", buffering=1, encoding="utf-8"
)
self.file.reconfigure(write_through=True)
self.file.seek(0)
self.fingerprints.update(x.rstrip() for x in self.file)