RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Warn only once for all spiders

This commit is contained in:
Paul Tremberth 2017-03-07 15:41:17 +01:00
parent 978306a223
commit 0b9a18e1a1
2 changed files with 21 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
scrapy/spiderloader.py
View File

@ -23,18 +23,21 @@ class SpiderLoader(object):
self._found = defaultdict(list)
self._load_all_spiders()
def _check_name_duplicates(self):
dupes = ["\n".join(" {cls} named {name!r} (in {module})".format(
module=mod, cls=cls, name=name)
for (mod, cls) in locations)
for name, locations in self._found.items()
if len(locations)>1]
if dupes:
msg = ("There are several spiders with the same name:\n\n"
"{}\n\n This can cause unexpected behavior.".format(
"\n\n".join(dupes)))
warnings.warn(msg, UserWarning)
def _load_spiders(self, module):
for spcls in iter_spider_classes(module):
self._found[spcls.name].append((module.__name__, spcls.__name__))
if spcls.name in self._spiders.keys():
import warnings
msg = ("There are several spiders with the same name {!r}:\n"
"{}\n This can cause unexpected behavior.".format(
spcls.name,
"\n".join(
" {1} (in {0})".format(mod, cls)
for (mod, cls) in self._found[spcls.name])))
warnings.warn(msg, UserWarning)
self._spiders[spcls.name] = spcls
def _load_all_spiders(self):
@ -47,6 +50,7 @@ class SpiderLoader(object):
"Check SPIDER_MODULES setting".format(
modname=name, tb=traceback.format_exc()))
warnings.warn(msg, RuntimeWarning)
self._check_name_duplicates()
@classmethod
def from_settings(cls, settings):

13
tests/test_spiderloader/__init__.py
View File

@ -127,7 +127,9 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
spider_loader = SpiderLoader.from_settings(self.settings)
self.assertEqual(len(w), 1)
self.assertIn("several spiders with the same name 'spider3'", str(w[0].message))
msg = str(w[0].message)
self.assertIn("several spiders with the same name", msg)
self.assertIn("'spider3'", msg)
spiders = set(spider_loader.list())
self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))
@ -143,10 +145,11 @@ class DuplicateSpiderNameLoaderTest(unittest.TestCase):
with warnings.catch_warnings(record=True) as w:
spider_loader = SpiderLoader.from_settings(self.settings)
self.assertEqual(len(w), 2)
msgs = sorted(str(wrn.message) for wrn in w)
self.assertIn("several spiders with the same name 'spider1'", msgs[0])
self.assertIn("several spiders with the same name 'spider2'", msgs[1])
self.assertEqual(len(w), 1)
msg = str(w[0].message)
self.assertIn("several spiders with the same name", msg)
self.assertIn("'spider1'", msg)
self.assertIn("'spider2'", msg)
spiders = set(spider_loader.list())
self.assertEqual(spiders, set(['spider1', 'spider2', 'spider3', 'spider4']))