From 701cd2ff9d4be34fcf63a7410d09f425410635f9 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 9 Oct 2017 09:42:34 -0300 Subject: [PATCH 1/5] Add from_crawler support to dupefilters --- scrapy/core/scheduler.py | 7 ++++++- tests/test_dupefilters.py | 21 +++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index a54b4daf0..faed27fd1 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -26,7 +26,12 @@ class Scheduler(object): def from_crawler(cls, crawler): settings = crawler.settings dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) - dupefilter = dupefilter_cls.from_settings(settings) + if hasattr(dupefilter_cls, 'from_crawler'): + dupefilter = dupefilter_cls.from_crawler(crawler) + elif hasattr(dupefilter_cls, 'from_settings'): + dupefilter = dupefilter_cls.from_settings(crawler.settings) + else: + dupefilter = dupefilter_cls() pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE']) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 2d1a4bfff..81524fddd 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -6,10 +6,31 @@ import shutil from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request from scrapy.utils.python import to_bytes +from scrapy.utils.job import job_dir +from scrapy.utils.test import get_crawler class RFPDupeFilterTest(unittest.TestCase): + def test_dupefilter_from_crawler(self): + + class FromCrawlerRFPDupeFilter(RFPDupeFilter): + + @classmethod + def from_crawler(cls, crawler): + debug = crawler.settings.getbool('DUPEFILTER_DEBUG') + df = cls(job_dir(crawler.settings), debug) + df.user_agent = crawler.settings.get('USER_AGENT') + return df + + crawler = get_crawler(settings_dict={'DUPEFILTER_DEBUG': True, 'USER_AGENT': 'test ua'}) + dupefilter = FromCrawlerRFPDupeFilter.from_crawler(crawler) + + self.assertTrue(dupefilter.debug) + self.assertEqual(dupefilter.user_agent, 'test ua') + + dupefilter.close('finished') + def test_filter(self): dupefilter = RFPDupeFilter() dupefilter.open() From d306fe30ac08401e74fd20cf90e5164a4125d8e1 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 30 Dec 2017 22:49:22 -0300 Subject: [PATCH 2/5] Test dupefilter creation by the Scheduler --- tests/test_dupefilters.py | 52 ++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 81524fddd..990b5141c 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -5,31 +5,49 @@ import shutil from scrapy.dupefilters import RFPDupeFilter from scrapy.http import Request +from scrapy.core.scheduler import Scheduler from scrapy.utils.python import to_bytes from scrapy.utils.job import job_dir from scrapy.utils.test import get_crawler +class FromCrawlerRFPDupeFilter(RFPDupeFilter): + + @classmethod + def from_crawler(cls, crawler): + debug = crawler.settings.getbool('DUPEFILTER_DEBUG') + df = cls(job_dir(crawler.settings), debug) + df.method = crawler.settings.get('METHOD') + return df + + +class FromSettingsRFPDupeFilter(RFPDupeFilter): + + @classmethod + def from_settings(cls, settings): + debug = settings.getbool('DUPEFILTER_DEBUG') + df = cls(job_dir(settings), debug) + df.method = settings.get('METHOD') + return df + + class RFPDupeFilterTest(unittest.TestCase): - def test_dupefilter_from_crawler(self): + def test_from_crawler_scheduler(self): + settings = {'DUPEFILTER_DEBUG': True, 'METHOD': 'from_crawler', + 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} + crawler = get_crawler(settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + self.assertTrue(scheduler.df.debug) + self.assertEqual(scheduler.df.method, 'from_crawler') - class FromCrawlerRFPDupeFilter(RFPDupeFilter): - - @classmethod - def from_crawler(cls, crawler): - debug = crawler.settings.getbool('DUPEFILTER_DEBUG') - df = cls(job_dir(crawler.settings), debug) - df.user_agent = crawler.settings.get('USER_AGENT') - return df - - crawler = get_crawler(settings_dict={'DUPEFILTER_DEBUG': True, 'USER_AGENT': 'test ua'}) - dupefilter = FromCrawlerRFPDupeFilter.from_crawler(crawler) - - self.assertTrue(dupefilter.debug) - self.assertEqual(dupefilter.user_agent, 'test ua') - - dupefilter.close('finished') + def test_from_settings_scheduler(self): + settings = {'DUPEFILTER_DEBUG': True, 'METHOD': 'from_settings', + 'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'} + crawler = get_crawler(settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + self.assertTrue(scheduler.df.debug) + self.assertEqual(scheduler.df.method, 'from_settings') def test_filter(self): dupefilter = RFPDupeFilter() From 0089a4ab31d1764dd38c30c4448a4c62efd9b9c3 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 23 Mar 2018 13:19:31 -0300 Subject: [PATCH 3/5] Add test for direct creation of dupefilter (no from_crawler/from_settings) --- tests/test_dupefilters.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 990b5141c..02a01fc94 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -17,7 +17,7 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter): def from_crawler(cls, crawler): debug = crawler.settings.getbool('DUPEFILTER_DEBUG') df = cls(job_dir(crawler.settings), debug) - df.method = crawler.settings.get('METHOD') + df.method = 'from_crawler' return df @@ -27,28 +27,40 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter): def from_settings(cls, settings): debug = settings.getbool('DUPEFILTER_DEBUG') df = cls(job_dir(settings), debug) - df.method = settings.get('METHOD') + df.method = 'from_settings' return df +class DirectRFPDupeFilter(RFPDupeFilter): + method = 'n/a' + + class RFPDupeFilterTest(unittest.TestCase): - def test_from_crawler_scheduler(self): - settings = {'DUPEFILTER_DEBUG': True, 'METHOD': 'from_crawler', + def test_df_from_crawler_scheduler(self): + settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, 'from_crawler') - def test_from_settings_scheduler(self): - settings = {'DUPEFILTER_DEBUG': True, 'METHOD': 'from_settings', + def test_df_from_settings_scheduler(self): + settings = {'DUPEFILTER_DEBUG': True, 'DUPEFILTER_CLASS': __name__ + '.FromSettingsRFPDupeFilter'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, 'from_settings') + def test_df_direct_scheduler(self): + settings = {'DUPEFILTER_DEBUG': True, + 'DUPEFILTER_CLASS': __name__ + '.DirectRFPDupeFilter'} + crawler = get_crawler(settings_dict=settings) + scheduler = Scheduler.from_crawler(crawler) + self.assertTrue(scheduler.df.debug) + self.assertEqual(scheduler.df.method, 'n/a') + def test_filter(self): dupefilter = RFPDupeFilter() dupefilter.open() From 9e14f8c7e4141fd5216efc378c9d003ea981a4d5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 23 Mar 2018 21:19:57 -0300 Subject: [PATCH 4/5] Fix test for dupefilter --- tests/test_dupefilters.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py index 02a01fc94..db69597a2 100644 --- a/tests/test_dupefilters.py +++ b/tests/test_dupefilters.py @@ -31,7 +31,7 @@ class FromSettingsRFPDupeFilter(RFPDupeFilter): return df -class DirectRFPDupeFilter(RFPDupeFilter): +class DirectDupeFilter(object): method = 'n/a' @@ -54,11 +54,9 @@ class RFPDupeFilterTest(unittest.TestCase): self.assertEqual(scheduler.df.method, 'from_settings') def test_df_direct_scheduler(self): - settings = {'DUPEFILTER_DEBUG': True, - 'DUPEFILTER_CLASS': __name__ + '.DirectRFPDupeFilter'} + settings = {'DUPEFILTER_CLASS': __name__ + '.DirectDupeFilter'} crawler = get_crawler(settings_dict=settings) scheduler = Scheduler.from_crawler(crawler) - self.assertTrue(scheduler.df.debug) self.assertEqual(scheduler.df.method, 'n/a') def test_filter(self): From 999341b60bd5289ca97c8187d0ff380a8555ba5e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 20 Jul 2018 22:17:55 -0300 Subject: [PATCH 5/5] Simplify dupefilter creation --- scrapy/core/scheduler.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index faed27fd1..eb790a67e 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -4,7 +4,7 @@ import logging from os.path import join, exists from scrapy.utils.reqser import request_to_dict, request_from_dict -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_object, create_instance from scrapy.utils.job import job_dir logger = logging.getLogger(__name__) @@ -26,12 +26,7 @@ class Scheduler(object): def from_crawler(cls, crawler): settings = crawler.settings dupefilter_cls = load_object(settings['DUPEFILTER_CLASS']) - if hasattr(dupefilter_cls, 'from_crawler'): - dupefilter = dupefilter_cls.from_crawler(crawler) - elif hasattr(dupefilter_cls, 'from_settings'): - dupefilter = dupefilter_cls.from_settings(crawler.settings) - else: - dupefilter = dupefilter_cls() + dupefilter = create_instance(dupefilter_cls, settings, crawler) pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE']) dqclass = load_object(settings['SCHEDULER_DISK_QUEUE']) mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])