Bump pylint, cleanup the ignored tags.

docs/conf.py

@@ -227,7 +227,7 @@ latex_documents = [
 # A list of regular expressions that match URIs that should not be checked when
 # doing a linkcheck build.
 linkcheck_ignore = [
-    "http://localhost:\d+",
+    r"http://localhost:\d+",
     "http://hg.scrapy.org",
     "http://directory.google.com/",
 ]

pylintrc

@@ -4,21 +4,14 @@ jobs=1  # >1 hides results
 
 [MESSAGES CONTROL]
 disable=abstract-method,
-        anomalous-backslash-in-string,
         arguments-differ,
         arguments-renamed,
         attribute-defined-outside-init,
         bad-classmethod-argument,
-        bad-mcs-classmethod-argument,
         bare-except,
         broad-except,
         broad-exception-raised,
         c-extension-no-member,
-        catching-non-exception,
-        cell-var-from-loop,
-        comparison-with-callable,
-        consider-using-dict-items,
-        consider-using-in,
         consider-using-with,
         cyclic-import,
         dangerous-default-value,
@@ -32,7 +25,6 @@ disable=abstract-method,
         implicit-str-concat,
         import-error,
         import-outside-toplevel,
-        import-self,
         inconsistent-return-statements,
         inherit-non-class,
         invalid-name,
@@ -44,7 +36,6 @@ disable=abstract-method,
         logging-fstring-interpolation,
         logging-not-lazy,
         lost-exception,
-        method-hidden,
         missing-docstring,
         no-else-raise,
         no-else-return,
@@ -52,7 +43,7 @@ disable=abstract-method,
         no-method-argument,
         no-name-in-module,
         no-self-argument,
-        no-value-for-parameter,
+        no-value-for-parameter,  # https://github.com/pylint-dev/pylint/issues/3268
         not-callable,
         pointless-exception-statement,
         pointless-statement,
@@ -77,14 +68,10 @@ disable=abstract-method,
         too-many-public-methods,
         too-many-return-statements,
         unbalanced-tuple-unpacking,
-        undefined-variable,
-        undefined-loop-variable,
-        unexpected-special-method-signature,
         unnecessary-comprehension,
         unnecessary-dunder-call,
         unnecessary-pass,
         unreachable,
-        unsubscriptable-object,
         unused-argument,
         unused-import,
         unused-private-member,
@@ -92,8 +79,6 @@ disable=abstract-method,
         unused-wildcard-import,
         use-dict-literal,
         used-before-assignment,
-        useless-object-inheritance,  # Required for Python 2 support
         useless-return,
-        useless-super-delegation,
         wildcard-import,
         wrong-import-position

scrapy/downloadermiddlewares/httpcompression.py

@@ -169,7 +169,7 @@ class HttpCompressionMiddleware:
         return to_decode, to_keep
 
     def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:
-        if encoding == b"gzip" or encoding == b"x-gzip":
+        if encoding in {b"gzip", b"x-gzip"}:
             return gunzip(body, max_size=max_size)
         if encoding == b"deflate":
             return _inflate(body, max_size=max_size)

scrapy/http/headers.py

@@ -113,7 +113,9 @@ class Headers(CaselessDict):
         return ((k, self.getlist(k)) for k in self.keys())
 
     def values(self) -> List[Optional[bytes]]:  # type: ignore[override]
-        return [self[k] for k in self.keys()]
+        return [
+            self[k] for k in self.keys()  # pylint: disable=consider-using-dict-items
+        ]
 
     def to_string(self) -> bytes:
         # cast() can be removed if the headers_dict_to_raw() hint is improved

scrapy/utils/ossignal.py

@@ -24,7 +24,11 @@ def install_shutdown_handlers(
     (e.g. Pdb)
     """
     signal.signal(signal.SIGTERM, function)
-    if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:
+    if (
+        signal.getsignal(signal.SIGINT)  # pylint: disable=comparison-with-callable
+        == signal.default_int_handler
+        or override_sigint
+    ):
         signal.signal(signal.SIGINT, function)
     # Catch Ctrl-Break in windows
     if hasattr(signal, "SIGBREAK"):

scrapy/utils/signal.py

@@ -98,7 +98,10 @@ def send_catch_log_deferred(
             robustApply, receiver, signal=signal, sender=sender, *arguments, **named
         )
         d.addErrback(logerror, receiver)
-        d.addBoth(lambda result: (receiver, result))
+        # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html
+        d.addBoth(
+            lambda result: (receiver, result)  # pylint: disable=cell-var-from-loop
+        )
         dfds.append(d)
     d = DeferredList(dfds)
     d.addCallback(lambda out: [x[1] for x in out])

tests/test_commands.py

@@ -991,38 +991,11 @@ class MySpider(scrapy.Spider):
 class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
     spider_filename = "myspider.pyw"
 
-    def setUp(self):
-        super().setUp()
-
     def test_start_requests_errors(self):
         log = self.get_log(self.badspider, name="badspider.pyw")
         self.assertIn("start_requests", log)
         self.assertIn("badspider.pyw", log)
 
-    def test_run_good_spider(self):
-        super().test_run_good_spider()
-
-    def test_runspider(self):
-        super().test_runspider()
-
-    def test_runspider_dnscache_disabled(self):
-        super().test_runspider_dnscache_disabled()
-
-    def test_runspider_log_level(self):
-        super().test_runspider_log_level()
-
-    def test_runspider_log_short_names(self):
-        super().test_runspider_log_short_names()
-
-    def test_runspider_no_spider_found(self):
-        super().test_runspider_no_spider_found()
-
-    def test_output(self):
-        super().test_output()
-
-    def test_overwrite_output(self):
-        super().test_overwrite_output()
-
     def test_runspider_unable_to_load(self):
         raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ")
 

tests/test_item.py

@@ -290,7 +290,9 @@ class ItemMetaTest(unittest.TestCase):
 class ItemMetaClassCellRegression(unittest.TestCase):
     def test_item_meta_classcell_regression(self):
         class MyItem(Item, metaclass=ItemMeta):
-            def __init__(self, *args, **kwargs):
+            def __init__(
+                self, *args, **kwargs
+            ):  # pylint: disable=useless-parent-delegation
                 # This call to super() trigger the __classcell__ propagation
                 # requirement. When not done properly raises an error:
                 # TypeError: __class__ set to <class '__main__.MyItem'>

tests/test_linkextractors.py

@@ -818,9 +818,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
             ],
         )
 
-    def test_restrict_xpaths_with_html_entities(self):
-        super().test_restrict_xpaths_with_html_entities()
-
     @mark.skipif(
         Version(w3lib_version) < Version("2.0.0"),
         reason=(

tests/test_loader_deprecated.py

@@ -678,11 +678,11 @@ class SelectJmesTestCase(unittest.TestCase):
     }
 
     def test_output(self):
-        for tl in self.test_list_equals:
-            expr, test_list, expected = self.test_list_equals[tl]
+        for k, v in self.test_list_equals.items():
+            expr, test_list, expected = v
             test = SelectJmes(expr)(test_list)
             self.assertEqual(
-                test, expected, msg=f'test "{tl}" got {test} expected {expected}'
+                test, expected, msg=f'test "{k}" got {test} expected {expected}'
             )
 
 

tests/test_settings/__init__.py

@@ -426,7 +426,7 @@ class SettingsTest(unittest.TestCase):
         mydict = settings.get("TEST_DICT")
         self.assertIsInstance(mydict, BaseSettings)
         self.assertIn("key", mydict)
-        self.assertEqual(mydict["key"], "val")
+        self.assertEqual(mydict["key"], "val")  # pylint: disable=unsubscriptable-object
         self.assertEqual(mydict.getpriority("key"), 0)
 
     @mock.patch("scrapy.settings.default_settings", default_settings)

tests/test_utils_datatypes.py

@@ -353,7 +353,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase):
         for i, r in enumerate(refs):
             self.assertIn(r, cache)
             self.assertEqual(cache[r], i)
-        del r  # delete reference to the last object in the list
+        del r  # delete reference to the last object in the list  # pylint: disable=undefined-loop-variable
 
         # delete half of the objects, make sure that is reflected in the cache
         for _ in range(max // 2):

tests/test_utils_signal.py

@@ -75,9 +75,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
         await defer.succeed(42)
         return "OK"
 
-    def test_send_catch_log(self):
-        return super().test_send_catch_log()
-
 
 @mark.only_asyncio()
 class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
@@ -87,9 +84,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
         await asyncio.sleep(0.2)
         return await get_from_asyncio_queue("OK")
 
-    def test_send_catch_log(self):
-        return super().test_send_catch_log()
-
 
 class SendCatchLogTest2(unittest.TestCase):
     def test_error_logged_if_deferred_not_supported(self):

tests/test_utils_spider.py

@@ -26,7 +26,7 @@ class UtilsSpidersTestCase(unittest.TestCase):
         self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o])
 
     def test_iter_spider_classes(self):
-        import tests.test_utils_spider
+        import tests.test_utils_spider  # pylint: disable=import-self
 
         it = iter_spider_classes(tests.test_utils_spider)
         self.assertEqual(set(it), {MySpider1, MySpider2})

tox.ini

@@ -53,7 +53,7 @@ commands =
 basepython = python3
 deps =
     {[testenv:extra-deps]deps}
-    pylint==3.0.1
+    pylint==3.1.0
 commands =
     pylint conftest.py docs extras scrapy setup.py tests