diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index a54b4daf0..faed27fd1 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -26,7 +26,12 @@ class Scheduler(object): def from_crawler(cls, crawler): settings = crawler.settings dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) - dupefilter = dupefilter_cls.from_settings(settings) + if hasattr(dupefilter_cls, 'from_crawler'): + dupefilter = dupefilter_cls.from_crawler(crawler) + elif hasattr(dupefilter_cls, 'from_settings'): + dupefilter = dupefilter_cls.from_settings(crawler.settings) + else: + dupefilter = dupefilter_cls() pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 2d1a4bfff..81524fddd 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -6,10 +6,31 @@ import shutil from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request from scrapy.utils.python import to_bytes +from scrapy.utils.job import job_dir +from scrapy.utils.test import get_crawler class RFPDupeFilterTest(unittest.TestCase): + def test_dupefilter_from_crawler(self): + + class FromCrawlerRFPDupeFilter(RFPDupeFilter): + + @classmethod + def from_crawler(cls, crawler): + debug = crawler.settings.getbool('DUPEFILTER_DEBUG') + df = cls(job_dir(crawler.settings), debug) + df.user_agent = crawler.settings.get('USER_AGENT') + return df + + crawler = get_crawler(settings_dict={'DUPEFILTER_DEBUG': True, 'USER_AGENT': 'test ua'}) + dupefilter = FromCrawlerRFPDupeFilter.from_crawler(crawler) + + self.assertTrue(dupefilter.debug) + self.assertEqual(dupefilter.user_agent, 'test ua') + + dupefilter.close('finished') + def test_filter(self): dupefilter = RFPDupeFilter() dupefilter.open()