Add from_crawler support to dupefilters

scrapy/core/scheduler.py

@@ -26,7 +26,12 @@ class Scheduler(object):
     def from_crawler(cls, crawler):
         settings = crawler.settings
         dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
-        dupefilter = dupefilter_cls.from_settings(settings)
+        if hasattr(dupefilter_cls, 'from_crawler'):
+            dupefilter = dupefilter_cls.from_crawler(crawler)
+        elif hasattr(dupefilter_cls, 'from_settings'):
+            dupefilter = dupefilter_cls.from_settings(crawler.settings)
+        else:
+            dupefilter = dupefilter_cls()
         pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
         dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
         mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])

tests/test_dupefilters.py

@@ -6,10 +6,31 @@ import shutil
 from scrapy.dupefilters import RFPDupeFilter
 from scrapy.http import Request
 from scrapy.utils.python import to_bytes
+from scrapy.utils.job import job_dir
+from scrapy.utils.test import get_crawler
 
 
 class RFPDupeFilterTest(unittest.TestCase):
 
+    def test_dupefilter_from_crawler(self):
+
+        class FromCrawlerRFPDupeFilter(RFPDupeFilter):
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
+                df = cls(job_dir(crawler.settings), debug)
+                df.user_agent = crawler.settings.get('USER_AGENT')
+                return df
+
+        crawler = get_crawler(settings_dict={'DUPEFILTER_DEBUG': True, 'USER_AGENT': 'test ua'})
+        dupefilter = FromCrawlerRFPDupeFilter.from_crawler(crawler)
+
+        self.assertTrue(dupefilter.debug)
+        self.assertEqual(dupefilter.user_agent, 'test ua')
+
+        dupefilter.close('finished')
+
     def test_filter(self):
         dupefilter = RFPDupeFilter()
         dupefilter.open()