diff --git a/docs/news.rst b/docs/news.rst index 758b22d80..58b51c9ea 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -8,6 +8,12 @@ Release notes Scrapy VERSION (YYYY-MM-DD) --------------------------- +New features +~~~~~~~~~~~~ + +- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``, + ``SpiderLoader`` does not raise :exc:`SyntaxError` but emits a warning instead. + Deprecations ~~~~~~~~~~~~ diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 904bd7ecc..02fca7ff4 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1580,7 +1580,7 @@ SPIDER_LOADER_WARN_ONLY Default: ``False`` By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`, -it will fail loudly if there is any ``ImportError`` exception. +it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception. But you can choose to silence this exception and turn it into a simple warning by setting ``SPIDER_LOADER_WARN_ONLY = True``. diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index b8fe65668..f5fd899b2 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -64,7 +64,7 @@ class SpiderLoader: try: for module in walk_modules(name): self._load_spiders(module) - except ImportError: + except (ImportError, SyntaxError): if self.warn_only: warnings.warn( f"\n{traceback.format_exc()}Could not load spiders " diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py index f950739f2..32699d837 100644 --- a/tests/test_spiderloader/__init__.py +++ b/tests/test_spiderloader/__init__.py @@ -4,6 +4,7 @@ import tempfile import warnings from pathlib import Path from tempfile import mkdtemp +from unittest import mock from twisted.trial import unittest from zope.interface.verify import verifyObject @@ -136,6 +137,33 @@ class SpiderLoaderTest(unittest.TestCase): spiders = spider_loader.list() self.assertEqual(spiders, []) + def test_syntax_error_exception(self): + module = "tests.test_spiderloader.test_spiders.spider1" + with mock.patch.object(SpiderLoader, "_load_spiders") as m: + m.side_effect = SyntaxError + settings = Settings({"SPIDER_MODULES": [module]}) + self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings) + + def test_syntax_error_warning(self): + with warnings.catch_warnings(record=True) as w, mock.patch.object( + SpiderLoader, "_load_spiders" + ) as m: + m.side_effect = SyntaxError + module = "tests.test_spiderloader.test_spiders.spider1" + settings = Settings( + {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True} + ) + spider_loader = SpiderLoader.from_settings(settings) + if str(w[0].message).startswith("_SixMetaPathImporter"): + # needed on 3.10 because of https://github.com/benjaminp/six/issues/349, + # at least until all six versions we can import (including botocore.vendored.six) + # are updated to 1.16.0+ + w.pop(0) + self.assertIn("Could not load spiders from module", str(w[0].message)) + + spiders = spider_loader.list() + self.assertEqual(spiders, []) + class DuplicateSpiderNameLoaderTest(unittest.TestCase): def setUp(self):