diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index a54b4daf0..eb790a67e 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,7 +4,7 @@ import logging from os.path import join, exists from scrapy.utils.reqser import request_to_dict, request_from_dict -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_object, create_instance from scrapy.utils.job import job_dir logger = logging.getLogger(__name__) @@ -26,7 +26,7 @@ class Scheduler(object): def from_crawler(cls, crawler): settings = crawler.settings dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) - dupefilter = dupefilter_cls.from_settings(settings) + dupefilter = create_instance(dupefilter_cls, settings, crawler) pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 2d1a4bfff..db69597a2 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -5,11 +5,60 @@ import shutil from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request +from scrapy.core.scheduler import Scheduler from scrapy.utils.python import to_bytes +from scrapy.utils.job import job_dir +from scrapy.utils.test import get_crawler + + +class FromCrawlerRFPDupeFilter(RFPDupeFilter): + + @classmethod + def from_crawler(cls, crawler): + debug = crawler.settings.getbool('DUPEFILTER_DEBUG') + df = cls(job_dir(crawler.settings), debug) + df.method = 'from_crawler' + return df + + +class FromSettingsRFPDupeFilter(RFPDupeFilter): + + @classmethod + def from_settings(cls, settings): + debug = settings.getbool('DUPEFILTER_DEBUG') + df = cls(job_dir(settings), debug) + df.method = 'from_settings' + return df + + +class DirectDupeFilter(object): + method = 'n/a' class RFPDupeFilterTest(unittest.TestCase): + def test_df_from_crawler_scheduler(self): + settings = {'DUPEFILTER_DEBUG': True, + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + crawler = get_crawler(settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + self.assertTrue(scheduler.df.debug) + self.assertEqual(scheduler.df.method, 'from_crawler') + + def test_df_from_settings_scheduler(self): + settings = {'DUPEFILTER_DEBUG': True, + 'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'} + crawler = get_crawler(settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + self.assertTrue(scheduler.df.debug) + self.assertEqual(scheduler.df.method, 'from_settings') + + def test_df_direct_scheduler(self): + settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'} + crawler = get_crawler(settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + self.assertEqual(scheduler.df.method, 'n/a') + def test_filter(self): dupefilter = RFPDupeFilter() dupefilter.open()