RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

rename CrawlerRunner.spiders to CrawlerRunner.spider_loader

This commit is contained in:
Mikhail Korobov 2015-04-17 01:54:26 +05:00
parent fee5565709
commit ad587ea792
12 changed files with 57 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/commands/check.py
View File

@ -69,10 +69,10 @@ class Command(ScrapyCommand):
# contract requests # contract requests
contract_reqs = defaultdict(list) contract_reqs = defaultdict(list)
spiders = self.crawler_process.spiders spider_loader = self.crawler_process.spider_loader
for spidername in args or spiders.list(): for spidername in args or spider_loader.list():
spidercls = spiders.load(spidername) spidercls = spider_loader.load(spidername)
spidercls.start_requests = lambda s: conman.from_spider(s, result) spidercls.start_requests = lambda s: conman.from_spider(s, result)
tested_methods = conman.tested_methods_from_spidercls(spidercls) tested_methods = conman.tested_methods_from_spidercls(spidercls)

2
scrapy/commands/edit.py
View File

@ -27,7 +27,7 @@ class Command(ScrapyCommand):
editor = self.settings['EDITOR'] editor = self.settings['EDITOR']
try: try:
spidercls = self.crawler_process.spiders.load(args[0]) spidercls = self.crawler_process.spider_loader.load(args[0])
except KeyError: except KeyError:
return self._err("Spider not found: %s" % args[0]) return self._err("Spider not found: %s" % args[0])

6
scrapy/commands/fetch.py
View File

@ -48,10 +48,10 @@ class Command(ScrapyCommand):
request.meta['handle_httpstatus_all'] = True request.meta['handle_httpstatus_all'] = True
spidercls = DefaultSpider spidercls = DefaultSpider
spiders = self.crawler_process.spiders spider_loader = self.crawler_process.spider_loader
if opts.spider: if opts.spider:
spidercls = spiders.load(opts.spider) spidercls = spider_loader.load(opts.spider)
else: else:
spidercls = spidercls_for_request(spiders, request, spidercls) spidercls = spidercls_for_request(spider_loader, request, spidercls)
self.crawler_process.crawl(spidercls, start_requests=lambda: [request]) self.crawler_process.crawl(spidercls, start_requests=lambda: [request])
self.crawler_process.start() self.crawler_process.start()

2
scrapy/commands/genspider.py
View File

@ -65,7 +65,7 @@ class Command(ScrapyCommand):
return return
try: try:
spidercls = self.crawler_process.spiders.load(name) spidercls = self.crawler_process.spider_loader.load(name)
except KeyError: except KeyError:
pass pass
else: else:

2
scrapy/commands/list.py
View File

@ -10,5 +10,5 @@ class Command(ScrapyCommand):
return "List available spiders" return "List available spiders"
def run(self, args, opts): def run(self, args, opts):
for s in sorted(self.crawler_process.spiders.list()): for s in sorted(self.crawler_process.spider_loader.list()):
print(s) print(s)

6
scrapy/commands/parse.py
View File

@ -124,15 +124,15 @@ class Command(ScrapyCommand):
level=log.ERROR, spider=spider.name) level=log.ERROR, spider=spider.name)
def set_spidercls(self, url, opts): def set_spidercls(self, url, opts):
spiders = self.crawler_process.spiders spider_loader = self.crawler_process.spider_loader
if opts.spider: if opts.spider:
try: try:
self.spidercls = spiders.load(opts.spider) self.spidercls = spider_loader.load(opts.spider)
except KeyError: except KeyError:
log.msg(format='Unable to find spider: %(spider)s', log.msg(format='Unable to find spider: %(spider)s',
level=log.ERROR, spider=opts.spider) level=log.ERROR, spider=opts.spider)
else: else:
self.spidercls = spidercls_for_request(spiders, Request(url)) self.spidercls = spidercls_for_request(spider_loader, Request(url))
if not self.spidercls: if not self.spidercls:
log.msg(format='Unable to find spider for: %(url)s', log.msg(format='Unable to find spider for: %(url)s',
level=log.ERROR, url=url) level=log.ERROR, url=url)

6
scrapy/commands/shell.py
View File

@ -42,13 +42,13 @@ class Command(ScrapyCommand):
def run(self, args, opts): def run(self, args, opts):
url = args[0] if args else None url = args[0] if args else None
spiders = self.crawler_process.spiders spider_loader = self.crawler_process.spider_loader
spidercls = DefaultSpider spidercls = DefaultSpider
if opts.spider: if opts.spider:
spidercls = spiders.load(opts.spider) spidercls = spider_loader.load(opts.spider)
elif url: elif url:
spidercls = spidercls_for_request(spiders, Request(url), spidercls = spidercls_for_request(spider_loader, Request(url),
spidercls, log_multiple=True) spidercls, log_multiple=True)
# The crawler is created this way since the Shell manually handles the # The crawler is created this way since the Shell manually handles the

13
scrapy/crawler.py
View File

@ -43,7 +43,7 @@ class Crawler(object):
def spiders(self): def spiders(self):
if not hasattr(self, '_spiders'): if not hasattr(self, '_spiders'):
warnings.warn("Crawler.spiders is deprecated, use " warnings.warn("Crawler.spiders is deprecated, use "
"CrawlerRunner.spiders or instantiate " "CrawlerRunner.spider_loader or instantiate "
"scrapy.spiderloader.SpiderLoader with your " "scrapy.spiderloader.SpiderLoader with your "
"settings.", "settings.",
category=ScrapyDeprecationWarning, stacklevel=2) category=ScrapyDeprecationWarning, stacklevel=2)
@ -84,10 +84,17 @@ class CrawlerRunner(object):
if isinstance(settings, dict): if isinstance(settings, dict):
settings = Settings(settings) settings = Settings(settings)
self.settings = settings self.settings = settings
self.spiders = _get_spider_loader(settings) self.spider_loader = _get_spider_loader(settings)
self.crawlers = set() self.crawlers = set()
self._active = set() self._active = set()
@property
def spiders(self):
warnings.warn("CrawlerRunner.spiders attribute is renamed to "
"CrawlerRunner.spider_loader.",
category=ScrapyDeprecationWarning, stacklevel=2)
return self.spider_loader
def crawl(self, crawler_or_spidercls, *args, **kwargs): def crawl(self, crawler_or_spidercls, *args, **kwargs):
crawler = crawler_or_spidercls crawler = crawler_or_spidercls
if not isinstance(crawler_or_spidercls, Crawler): if not isinstance(crawler_or_spidercls, Crawler):
@ -107,7 +114,7 @@ class CrawlerRunner(object):
def _create_crawler(self, spidercls): def _create_crawler(self, spidercls):
if isinstance(spidercls, six.string_types): if isinstance(spidercls, six.string_types):
spidercls = self.spiders.load(spidercls) spidercls = self.spider_loader.load(spidercls)
return Crawler(spidercls, self.settings) return Crawler(spidercls, self.settings)
def _setup_crawler_logging(self, crawler): def _setup_crawler_logging(self, crawler):

2
scrapy/spiderloader.py
View File

@ -41,7 +41,7 @@ class SpiderLoader(object):
def find_by_request(self, request): def find_by_request(self, request):
""" """
Return the list of spiders names that can handle the given request. Return the list of spider names that can handle the given request.
""" """
return [name for name, cls in self._spiders.items() return [name for name, cls in self._spiders.items()
if cls.handles_request(request)] if cls.handles_request(request)]

6
scrapy/utils/spider.py
View File

@ -26,7 +26,7 @@ def iter_spider_classes(module):
getattr(obj, 'name', None): getattr(obj, 'name', None):
yield obj yield obj
def spidercls_for_request(spiderloader, request, default_spidercls=None, def spidercls_for_request(spider_loader, request, default_spidercls=None,
log_none=False, log_multiple=False): log_none=False, log_multiple=False):
"""Return a spider class that handles the given Request. """Return a spider class that handles the given Request.
@ -38,9 +38,9 @@ def spidercls_for_request(spiderloader, request, default_spidercls=None,
default_spidercls passed. It can optionally log if multiple or no spiders default_spidercls passed. It can optionally log if multiple or no spiders
are found. are found.
""" """
snames = spiderloader.find_by_request(request) snames = spider_loader.find_by_request(request)
if len(snames) == 1: if len(snames) == 1:
return spiderloader.load(snames[0]) return spider_loader.load(snames[0])
if len(snames) > 1 and log_multiple: if len(snames) > 1 and log_multiple:
log.msg(format='More than one spider can handle: %(request)s - %(snames)s', log.msg(format='More than one spider can handle: %(request)s - %(snames)s',

13
tests/test_crawler.py
View File

@ -19,8 +19,8 @@ class CrawlerTestCase(unittest.TestCase):
spiders = self.crawler.spiders spiders = self.crawler.spiders
self.assertEqual(len(w), 1) self.assertEqual(len(w), 1)
self.assertIn("Crawler.spiders", str(w[0].message)) self.assertIn("Crawler.spiders", str(w[0].message))
sm_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS']) sl_cls = load_object(self.crawler.settings['SPIDER_LOADER_CLASS'])
self.assertIsInstance(spiders, sm_cls) self.assertIsInstance(spiders, sl_cls)
self.crawler.spiders self.crawler.spiders
self.assertEqual(len(w), 1, "Warn deprecated access only once") self.assertEqual(len(w), 1, "Warn deprecated access only once")
@ -78,3 +78,12 @@ class CrawlerRunnerTestCase(unittest.TestCase):
) )
self.assertIsInstance(runner.settings, Settings) self.assertIsInstance(runner.settings, Settings)
def test_deprecated_attribute_spiders(self):
with warnings.catch_warnings(record=True) as w:
runner = CrawlerRunner(Settings())
spiders = runner.spiders
self.assertEqual(len(w), 1)
self.assertIn("CrawlerRunner.spiders", str(w[0].message))
self.assertIn("CrawlerRunner.spider_loader", str(w[0].message))
sl_cls = load_object(runner.settings['SPIDER_LOADER_CLASS'])
self.assertIsInstance(spiders, sl_cls)

34
tests/test_spiderloader/__init__.py
View File

@ -26,53 +26,53 @@ class SpiderLoaderTest(unittest.TestCase):
shutil.copytree(orig_spiders_dir, self.spiders_dir) shutil.copytree(orig_spiders_dir, self.spiders_dir)
sys.path.append(self.tmpdir) sys.path.append(self.tmpdir)
settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']}) settings = Settings({'SPIDER_MODULES': ['test_spiders_xxx']})
self.spiderloader = SpiderLoader.from_settings(settings) self.spider_loader = SpiderLoader.from_settings(settings)
def tearDown(self): def tearDown(self):
del self.spiderloader del self.spider_loader
del sys.modules['test_spiders_xxx'] del sys.modules['test_spiders_xxx']
sys.path.remove(self.tmpdir) sys.path.remove(self.tmpdir)
def test_interface(self): def test_interface(self):
verifyObject(ISpiderLoader, self.spiderloader) verifyObject(ISpiderLoader, self.spider_loader)
def test_list(self): def test_list(self):
self.assertEqual(set(self.spiderloader.list()), self.assertEqual(set(self.spider_loader.list()),
set(['spider1', 'spider2', 'spider3'])) set(['spider1', 'spider2', 'spider3']))
def test_load(self): def test_load(self):
spider1 = self.spiderloader.load("spider1") spider1 = self.spider_loader.load("spider1")
self.assertEqual(spider1.__name__, 'Spider1') self.assertEqual(spider1.__name__, 'Spider1')
def test_find_by_request(self): def test_find_by_request(self):
self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy1.org/test')), self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy1.org/test')),
['spider1']) ['spider1'])
self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy2.org/test')), self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy2.org/test')),
['spider2']) ['spider2'])
self.assertEqual(set(self.spiderloader.find_by_request(Request('http://scrapy3.org/test'))), self.assertEqual(set(self.spider_loader.find_by_request(Request('http://scrapy3.org/test'))),
set(['spider1', 'spider2'])) set(['spider1', 'spider2']))
self.assertEqual(self.spiderloader.find_by_request(Request('http://scrapy999.org/test')), self.assertEqual(self.spider_loader.find_by_request(Request('http://scrapy999.org/test')),
[]) [])
self.assertEqual(self.spiderloader.find_by_request(Request('http://spider3.com')), self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com')),
[]) [])
self.assertEqual(self.spiderloader.find_by_request(Request('http://spider3.com/onlythis')), self.assertEqual(self.spider_loader.find_by_request(Request('http://spider3.com/onlythis')),
['spider3']) ['spider3'])
def test_load_spider_module(self): def test_load_spider_module(self):
module = 'tests.test_spiderloader.test_spiders.spider1' module = 'tests.test_spiderloader.test_spiders.spider1'
settings = Settings({'SPIDER_MODULES': [module]}) settings = Settings({'SPIDER_MODULES': [module]})
self.spiderloader = SpiderLoader.from_settings(settings) self.spider_loader = SpiderLoader.from_settings(settings)
assert len(self.spiderloader._spiders) == 1 assert len(self.spider_loader._spiders) == 1
def test_load_spider_module(self): def test_load_spider_module(self):
prefix = 'tests.test_spiderloader.test_spiders.' prefix = 'tests.test_spiderloader.test_spiders.'
module = ','.join(prefix + s for s in ('spider1', 'spider2')) module = ','.join(prefix + s for s in ('spider1', 'spider2'))
settings = Settings({'SPIDER_MODULES': module}) settings = Settings({'SPIDER_MODULES': module})
self.spiderloader = SpiderLoader.from_settings(settings) self.spider_loader = SpiderLoader.from_settings(settings)
assert len(self.spiderloader._spiders) == 2 assert len(self.spider_loader._spiders) == 2
def test_load_base_spider(self): def test_load_base_spider(self):
module = 'tests.test_spiderloader.test_spiders.spider0' module = 'tests.test_spiderloader.test_spiders.spider0'
settings = Settings({'SPIDER_MODULES': [module]}) settings = Settings({'SPIDER_MODULES': [module]})
self.spiderloader = SpiderLoader.from_settings(settings) self.spider_loader = SpiderLoader.from_settings(settings)
assert len(self.spiderloader._spiders) == 0 assert len(self.spider_loader._spiders) == 0