Ignore SyntaxError as well when SPIDER_LOADER_WARN_ONLY is set to True (#6484)

docs/news.rst

@@ -8,6 +8,12 @@ Release notes
 Scrapy VERSION (YYYY-MM-DD)
 ---------------------------
 
+New features
+~~~~~~~~~~~~
+
+- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``,
+  ``SpiderLoader`` does not raise :exc:`SyntaxError` but emits a warning instead.
+
 Deprecations
 ~~~~~~~~~~~~
 

docs/topics/settings.rst

@@ -1580,7 +1580,7 @@ SPIDER_LOADER_WARN_ONLY
 Default: ``False``
 
 By default, when Scrapy tries to import spider classes from :setting:`SPIDER_MODULES`,
-it will fail loudly if there is any ``ImportError`` exception.
+it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception.
 But you can choose to silence this exception and turn it into a simple
 warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
 

scrapy/spiderloader.py

@@ -64,7 +64,7 @@ class SpiderLoader:
             try:
                 for module in walk_modules(name):
                     self._load_spiders(module)
-            except ImportError:
+            except (ImportError, SyntaxError):
                 if self.warn_only:
                     warnings.warn(
                         f"\n{traceback.format_exc()}Could not load spiders "

tests/test_spiderloader/__init__.py

@@ -4,6 +4,7 @@ import tempfile
 import warnings
 from pathlib import Path
 from tempfile import mkdtemp
+from unittest import mock
 
 from twisted.trial import unittest
 from zope.interface.verify import verifyObject
@@ -136,6 +137,33 @@ class SpiderLoaderTest(unittest.TestCase):
             spiders = spider_loader.list()
             self.assertEqual(spiders, [])
 
+    def test_syntax_error_exception(self):
+        module = "tests.test_spiderloader.test_spiders.spider1"
+        with mock.patch.object(SpiderLoader, "_load_spiders") as m:
+            m.side_effect = SyntaxError
+            settings = Settings({"SPIDER_MODULES": [module]})
+            self.assertRaises(SyntaxError, SpiderLoader.from_settings, settings)
+
+    def test_syntax_error_warning(self):
+        with warnings.catch_warnings(record=True) as w, mock.patch.object(
+            SpiderLoader, "_load_spiders"
+        ) as m:
+            m.side_effect = SyntaxError
+            module = "tests.test_spiderloader.test_spiders.spider1"
+            settings = Settings(
+                {"SPIDER_MODULES": [module], "SPIDER_LOADER_WARN_ONLY": True}
+            )
+            spider_loader = SpiderLoader.from_settings(settings)
+            if str(w[0].message).startswith("_SixMetaPathImporter"):
+                # needed on 3.10 because of https://github.com/benjaminp/six/issues/349,
+                # at least until all six versions we can import (including botocore.vendored.six)
+                # are updated to 1.16.0+
+                w.pop(0)
+            self.assertIn("Could not load spiders from module", str(w[0].message))
+
+            spiders = spider_loader.list()
+            self.assertEqual(spiders, [])
+
 
 class DuplicateSpiderNameLoaderTest(unittest.TestCase):
     def setUp(self):