From c78e2636f944c291c875108c32bfe229112ad9fa Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 28 Jul 2015 14:31:42 +0500 Subject: [PATCH 1/2] TST a test to show that dupefilter persistence is not working --- tests/test_dupefilters.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index f8f800a72..a9d92a1a2 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -1,5 +1,7 @@ import hashlib +import tempfile import unittest +import shutil from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request @@ -23,6 +25,27 @@ class RFPDupeFilterTest(unittest.TestCase): dupefilter.close('finished') + def test_dupefilter_path(self): + r1 = Request('http://scrapytest.org/1') + r2 = Request('http://scrapytest.org/2') + + path = tempfile.mkdtemp() + try: + df = RFPDupeFilter(path) + df.open() + assert not df.request_seen(r1) + assert df.request_seen(r1) + df.close('finished') + + df2 = RFPDupeFilter(path) + df2.open() + assert df2.request_seen(r1) + assert not df2.request_seen(r2) + assert df2.request_seen(r2) + df2.close('finished') + finally: + shutil.rmtree(path) + def test_request_fingerprint(self): """Test if customization of request_fingerprint method will change output of request_seen. From 836cb84b7a78f7d8ea439362234387d799c06b5b Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 28 Jul 2015 14:32:14 +0500 Subject: [PATCH 2/2] fixed RFPDupeFilter persistence --- scrapy/dupefilters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index 739ba922f..9d8966b9c 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -36,6 +36,7 @@ class RFPDupeFilter(BaseDupeFilter): self.logger = logging.getLogger(__name__) if path: self.file = open(os.path.join(path, 'requests.seen'), 'a+') + self.file.seek(0) self.fingerprints.update(x.rstrip() for x in self.file) @classmethod