From ad587ea792d4f401abc69d010b350dcafe84eab6 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Fri, 17 Apr 2015 01:54:26 +0500 Subject: [PATCH] rename CrawlerRunner.spiders to CrawlerRunner.spider_loader --- scrapy/commands/check.py | 6 ++--- scrapy/commands/edit.py | 2 +- scrapy/commands/fetch.py | 6 ++--- scrapy/commands/genspider.py | 2 +- scrapy/commands/list.py | 2 +- scrapy/commands/parse.py | 6 ++--- scrapy/commands/shell.py | 6 ++--- scrapy/crawler.py | 13 ++++++++--- scrapy/spiderloader.py | 2 +- scrapy/utils/spider.py | 6 ++--- tests/test_crawler.py | 13 +++++++++-- tests/test_spiderloader/__init__.py | 34 ++++++++++++++--------------- 12 files changed, 57 insertions(+), 41 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 014b00eeb..bf07f360f 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -69,10 +69,10 @@ class Command(ScrapyCommand): # contract requests contract_reqs = defaultdict(list) - spiders = self.crawler_process.spiders + spider_loader = self.crawler_process.spider_loader - for spidername in args or spiders.list(): - spidercls = spiders.load(spidername) + for spidername in args or spider_loader.list(): + spidercls = spider_loader.load(spidername) spidercls.start_requests = lambda s: conman.from_spider(s, result) tested_methods = conman.tested_methods_from_spidercls(spidercls) diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index e20e7c2e5..016c98d2d 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -27,7 +27,7 @@ class Command(ScrapyCommand): editor = self.settings['EDITOR'] try: - spidercls = self.crawler_process.spiders.load(args[0]) + spidercls = self.crawler_process.spider_loader.load(args[0]) except KeyError: return self._err("Spider not found: %s" % args[0]) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index ca9fd57f5..89809a177 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -48,10 +48,10 @@ class Command(ScrapyCommand): request.meta['handle_httpstatus_all'] = True spidercls = DefaultSpider - spiders = self.crawler_process.spiders + spider_loader = self.crawler_process.spider_loader if opts.spider: - spidercls = spiders.load(opts.spider) + spidercls = spider_loader.load(opts.spider) else: - spidercls = spidercls_for_request(spiders, request, spidercls) + spidercls = spidercls_for_request(spider_loader, request, spidercls) self.crawler_process.crawl(spidercls, start_requests=lambda: [request]) self.crawler_process.start() diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 52c5d9f94..9d68ec845 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -65,7 +65,7 @@ class Command(ScrapyCommand): return try: - spidercls = self.crawler_process.spiders.load(name) + spidercls = self.crawler_process.spider_loader.load(name) except KeyError: pass else: diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index 2d55d59bd..3b35cbb92 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -10,5 +10,5 @@ class Command(ScrapyCommand): return "List available spiders" def run(self, args, opts): - for s in sorted(self.crawler_process.spiders.list()): + for s in sorted(self.crawler_process.spider_loader.list()): print(s) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index b8cc140d4..3e006ede3 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -124,15 +124,15 @@ class Command(ScrapyCommand): level=log.ERROR, spider=spider.name) def set_spidercls(self, url, opts): - spiders = self.crawler_process.spiders + spider_loader = self.crawler_process.spider_loader if opts.spider: try: - self.spidercls = spiders.load(opts.spider) + self.spidercls = spider_loader.load(opts.spider) except KeyError: log.msg(format='Unable to find spider: %(spider)s', level=log.ERROR, spider=opts.spider) else: - self.spidercls = spidercls_for_request(spiders, Request(url)) + self.spidercls = spidercls_for_request(spider_loader, Request(url)) if not self.spidercls: log.msg(format='Unable to find spider for: %(url)s', level=log.ERROR, url=url) diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 7c0706482..f8ad8a491 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -42,13 +42,13 @@ class Command(ScrapyCommand): def run(self, args, opts): url = args[0] if args else None - spiders = self.crawler_process.spiders + spider_loader = self.crawler_process.spider_loader spidercls = DefaultSpider if opts.spider: - spidercls = spiders.load(opts.spider) + spidercls = spider_loader.load(opts.spider) elif url: - spidercls = spidercls_for_request(spiders, Request(url), + spidercls = spidercls_for_request(spider_loader, Request(url), spidercls, log_multiple=True) # The crawler is created this way since the Shell manually handles the diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 52956d111..dab1043ec 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -43,7 +43,7 @@ class Crawler(object): def spiders(self): if not hasattr(self, '_spiders'): warnings.warn("Crawler.spiders is deprecated, use " - "CrawlerRunner.spiders or instantiate " + "CrawlerRunner.spider_loader or instantiate " "scrapy.spiderloader.SpiderLoader with your " "settings.", category=ScrapyDeprecationWarning, stacklevel=2) @@ -84,10 +84,17 @@ class CrawlerRunner(object): if isinstance(settings, dict): settings = Settings(settings) self.settings = settings - self.spiders = _get_spider_loader(settings) + self.spider_loader = _get_spider_loader(settings) self.crawlers = set() self._active = set() + @property + def spiders(self): + warnings.warn("CrawlerRunner.spiders attribute is renamed to " + "CrawlerRunner.spider_loader.", + category=ScrapyDeprecationWarning, stacklevel=2) + return self.spider_loader + def crawl(self, crawler_or_spidercls, *args, **kwargs): crawler = crawler_or_spidercls if not isinstance(crawler_or_spidercls, Crawler): @@ -107,7 +114,7 @@ class CrawlerRunner(object): def _create_crawler(self, spidercls): if isinstance(spidercls, six.string_types): - spidercls = self.spiders.load(spidercls) + spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) def _setup_crawler_logging(self, crawler): diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index 484fef456..a88db27c8 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -41,7 +41,7 @@ class SpiderLoader(object): def find_by_request(self, request): """ - Return the list of spiders names that can handle the given request. + Return the list of spider names that can handle the given request. """ return [name for name, cls in self._spiders.items() if cls.handles_request(request)] diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 68de76bfe..44f098f05 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -26,7 +26,7 @@ def iter_spider_classes(module): getattr(obj, 'name', None): yield obj -def spidercls_for_request(spiderloader, request, default_spidercls=None, +def spidercls_for_request(spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False): """Return a spider class that handles the given Request. @@ -38,9 +38,9 @@ def spidercls_for_request(spiderloader, request, default_spidercls=None, default_spidercls passed. It can optionally log if multiple or no spiders are found. """ - snames = spiderloader.find_by_request(request) + snames = spider_loader.find_by_request(request) if len(snames) == 1: - return spiderloader.load(snames[0]) + return spider_loader.load(snames[0]) if len(snames) > 1 and log_multiple: log.msg(format='More than one spider can handle: %(request)s - %(snames)s', diff --git a/tests/test_crawler.py b/tests/test_crawler.py index f0a3865cb..50da01e73 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -19,8 +19,8 @@ class CrawlerTestCase(unittest.TestCase): spiders = self.crawler.spiders self.assertEqual(len(w), 1) self.assertIn("Crawler.spiders", str(w[0].message)) - sm_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS']) - self.assertIsInstance(spiders, sm_cls) + sl_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS']) + self.assertIsInstance(spiders, sl_cls) self.crawler.spiders self.assertEqual(len(w), 1, "Warn deprecated access only once") @@ -78,3 +78,12 @@ class CrawlerRunnerTestCase(unittest.TestCase): ) self.assertIsInstance(runner.settings, Settings) + def test_deprecated_attribute_spiders(self): + with warnings.catch_warnings(record=True) as w: + runner = CrawlerRunner(Settings()) + spiders = runner.spiders + self.assertEqual(len(w), 1) + self.assertIn("CrawlerRunner.spiders", str(w[0].message)) + self.assertIn("CrawlerRunner.spider_loader", str(w[0].message)) + sl_cls = load_object(runner.settings['SPIDER_LOADER_CLASS']) + self.assertIsInstance(spiders, sl_cls) diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index aa3aea19d..42f2f29b3 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -26,53 +26,53 @@ class SpiderLoaderTest(unittest.TestCase): shutil.copytree(orig_spiders_dir, self.spiders_dir) sys.path.append(self.tmpdir) settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) - self.spiderloader = SpiderLoader.from_settings(settings) + self.spider_loader = SpiderLoader.from_settings(settings) def tearDown(self): - del self.spiderloader + del self.spider_loader del sys.modules['test_spiders_xxx'] sys.path.remove(self.tmpdir) def test_interface(self): - verifyObject(ISpiderLoader, self.spiderloader) + verifyObject(ISpiderLoader, self.spider_loader) def test_list(self): - self.assertEqual(set(self.spiderloader.list()), + self.assertEqual(set(self.spider_loader.list()), set(['spider1', 'spider2', 'spider3'])) def test_load(self): - spider1 = self.spiderloader.load("spider1") + spider1 = self.spider_loader.load("spider1") self.assertEqual(spider1.__name__, 'Spider1') def test_find_by_request(self): - self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy1.org/test')), + self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy1.org/test')), ['spider1']) - self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy2.org/test')), + self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy2.org/test')), ['spider2']) - self.assertEqual(set(self.spiderloader.find_by_request(Request('http://scrapy3.org/test'))), + self.assertEqual(set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))), set(['spider1', 'spider2'])) - self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy999.org/test')), + self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy999.org/test')), []) - self.assertEqual(self.spiderloader.find_by_request(Request('http://spider3.com')), + self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com')), []) - self.assertEqual(self.spiderloader.find_by_request(Request('http://spider3.com/onlythis')), + self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')), ['spider3']) def test_load_spider_module(self): module = 'tests.test_spiderloader.test_spiders.spider1' settings = Settings({'SPIDER_MODULES': [module]}) - self.spiderloader = SpiderLoader.from_settings(settings) - assert len(self.spiderloader._spiders) == 1 + self.spider_loader = SpiderLoader.from_settings(settings) + assert len(self.spider_loader._spiders) == 1 def test_load_spider_module(self): prefix = 'tests.test_spiderloader.test_spiders.' module = ','.join(prefix + s for s in ('spider1', 'spider2')) settings = Settings({'SPIDER_MODULES': module}) - self.spiderloader = SpiderLoader.from_settings(settings) - assert len(self.spiderloader._spiders) == 2 + self.spider_loader = SpiderLoader.from_settings(settings) + assert len(self.spider_loader._spiders) == 2 def test_load_base_spider(self): module = 'tests.test_spiderloader.test_spiders.spider0' settings = Settings({'SPIDER_MODULES': [module]}) - self.spiderloader = SpiderLoader.from_settings(settings) - assert len(self.spiderloader._spiders) == 0 + self.spider_loader = SpiderLoader.from_settings(settings) + assert len(self.spider_loader._spiders) == 0