Update RFPDupeFilter line separator for correct universal newlines mode usage (#4283)

scrapy/dupefilters.py

@@ -49,7 +49,7 @@ class RFPDupeFilter(BaseDupeFilter):
             return True
         self.fingerprints.add(fp)
         if self.file:
-            self.file.write(fp + os.linesep)
+            self.file.write(fp + '\n')
 
     def request_fingerprint(self, request):
         return request_fingerprint(request)

tests/test_dupefilters.py

@@ -2,6 +2,8 @@ import hashlib
 import tempfile
 import unittest
 import shutil
+import os
+import sys
 from testfixtures import LogCapture
 
 from scrapy.dupefilters import RFPDupeFilter
@@ -84,17 +86,21 @@ class RFPDupeFilterTest(unittest.TestCase):
         path = tempfile.mkdtemp()
         try:
             df = RFPDupeFilter(path)
-            df.open()
-            assert not df.request_seen(r1)
-            assert df.request_seen(r1)
-            df.close('finished')
+            try:
+                df.open()
+                assert not df.request_seen(r1)
+                assert df.request_seen(r1)
+            finally:
+                df.close('finished')
 
             df2 = RFPDupeFilter(path)
-            df2.open()
-            assert df2.request_seen(r1)
-            assert not df2.request_seen(r2)
-            assert df2.request_seen(r2)
-            df2.close('finished')
+            try:
+                df2.open()
+                assert df2.request_seen(r1)
+                assert not df2.request_seen(r2)
+                assert df2.request_seen(r2)
+            finally:
+                df2.close('finished')
         finally:
             shutil.rmtree(path)
 
@@ -129,6 +135,30 @@ class RFPDupeFilterTest(unittest.TestCase):
 
         case_insensitive_dupefilter.close('finished')
 
+    def test_seenreq_newlines(self):
+        """ Checks against adding duplicate \r to
+        line endings on Windows platforms. """
+
+        r1 = Request('http://scrapytest.org/1')
+
+        path = tempfile.mkdtemp()
+        try:
+            df = RFPDupeFilter(path)
+            df.open()
+            df.request_seen(r1)
+            df.close('finished')
+
+            with open(os.path.join(path, 'requests.seen'), 'rb') as seen_file:
+                line = next(seen_file).decode()
+                assert not line.endswith('\r\r\n')
+                if sys.platform == 'win32':
+                    assert line.endswith('\r\n')
+                else:
+                    assert line.endswith('\n')
+
+        finally:
+            shutil.rmtree(path)
+
     def test_log(self):
         with LogCapture() as l:
             settings = {'DUPEFILTER_DEBUG': False,