diff --git a/docs/conf.py b/docs/conf.py index 9ca0f817a..399078010 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -227,7 +227,7 @@ latex_documents = [ # A list of regular expressions that match URIs that should not be checked when # doing a linkcheck build. linkcheck_ignore = [ - "http://localhost:\d+", + r"http://localhost:\d+", "http://hg.scrapy.org", "http://directory.google.com/", ] diff --git a/pylintrc b/pylintrc index c8654b8d3..78004e78a 100644 --- a/pylintrc +++ b/pylintrc @@ -4,21 +4,14 @@ jobs=1 # >1 hides results [MESSAGES CONTROL] disable=abstract-method, - anomalous-backslash-in-string, arguments-differ, arguments-renamed, attribute-defined-outside-init, bad-classmethod-argument, - bad-mcs-classmethod-argument, bare-except, broad-except, broad-exception-raised, c-extension-no-member, - catching-non-exception, - cell-var-from-loop, - comparison-with-callable, - consider-using-dict-items, - consider-using-in, consider-using-with, cyclic-import, dangerous-default-value, @@ -32,7 +25,6 @@ disable=abstract-method, implicit-str-concat, import-error, import-outside-toplevel, - import-self, inconsistent-return-statements, inherit-non-class, invalid-name, @@ -44,7 +36,6 @@ disable=abstract-method, logging-fstring-interpolation, logging-not-lazy, lost-exception, - method-hidden, missing-docstring, no-else-raise, no-else-return, @@ -52,7 +43,7 @@ disable=abstract-method, no-method-argument, no-name-in-module, no-self-argument, - no-value-for-parameter, + no-value-for-parameter, # https://github.com/pylint-dev/pylint/issues/3268 not-callable, pointless-exception-statement, pointless-statement, @@ -77,14 +68,10 @@ disable=abstract-method, too-many-public-methods, too-many-return-statements, unbalanced-tuple-unpacking, - undefined-variable, - undefined-loop-variable, - unexpected-special-method-signature, unnecessary-comprehension, unnecessary-dunder-call, unnecessary-pass, unreachable, - unsubscriptable-object, unused-argument, unused-import, unused-private-member, @@ -92,8 +79,6 @@ disable=abstract-method, unused-wildcard-import, use-dict-literal, used-before-assignment, - useless-object-inheritance, # Required for Python 2 support useless-return, - useless-super-delegation, wildcard-import, wrong-import-position diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 1e340abb6..f0ad24f72 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -169,7 +169,7 @@ class HttpCompressionMiddleware: return to_decode, to_keep def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes: - if encoding == b"gzip" or encoding == b"x-gzip": + if encoding in {b"gzip", b"x-gzip"}: return gunzip(body, max_size=max_size) if encoding == b"deflate": return _inflate(body, max_size=max_size) diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py index 21eb9fb73..73aee7178 100644 --- a/scrapy/http/headers.py +++ b/scrapy/http/headers.py @@ -113,7 +113,9 @@ class Headers(CaselessDict): return ((k, self.getlist(k)) for k in self.keys()) def values(self) -> List[Optional[bytes]]: # type: ignore[override] - return [self[k] for k in self.keys()] + return [ + self[k] for k in self.keys() # pylint: disable=consider-using-dict-items + ] def to_string(self) -> bytes: # cast() can be removed if the headers_dict_to_raw() hint is improved diff --git a/scrapy/utils/ossignal.py b/scrapy/utils/ossignal.py index db9a71273..5985a847e 100644 --- a/scrapy/utils/ossignal.py +++ b/scrapy/utils/ossignal.py @@ -24,7 +24,11 @@ def install_shutdown_handlers( (e.g. Pdb) """ signal.signal(signal.SIGTERM, function) - if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint: + if ( + signal.getsignal(signal.SIGINT) # pylint: disable=comparison-with-callable + == signal.default_int_handler + or override_sigint + ): signal.signal(signal.SIGINT, function) # Catch Ctrl-Break in windows if hasattr(signal, "SIGBREAK"): diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index a25100c03..89cfbd2ec 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -98,7 +98,10 @@ def send_catch_log_deferred( robustApply, receiver, signal=signal, sender=sender, *arguments, **named ) d.addErrback(logerror, receiver) - d.addBoth(lambda result: (receiver, result)) + # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html + d.addBoth( + lambda result: (receiver, result) # pylint: disable=cell-var-from-loop + ) dfds.append(d) d = DeferredList(dfds) d.addCallback(lambda out: [x[1] for x in out]) diff --git a/tests/test_commands.py b/tests/test_commands.py index 2f36baa87..febad21da 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -991,38 +991,11 @@ class MySpider(scrapy.Spider): class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = "myspider.pyw" - def setUp(self): - super().setUp() - def test_start_requests_errors(self): log = self.get_log(self.badspider, name="badspider.pyw") self.assertIn("start_requests", log) self.assertIn("badspider.pyw", log) - def test_run_good_spider(self): - super().test_run_good_spider() - - def test_runspider(self): - super().test_runspider() - - def test_runspider_dnscache_disabled(self): - super().test_runspider_dnscache_disabled() - - def test_runspider_log_level(self): - super().test_runspider_log_level() - - def test_runspider_log_short_names(self): - super().test_runspider_log_short_names() - - def test_runspider_no_spider_found(self): - super().test_runspider_no_spider_found() - - def test_output(self): - super().test_output() - - def test_overwrite_output(self): - super().test_overwrite_output() - def test_runspider_unable_to_load(self): raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ") diff --git a/tests/test_item.py b/tests/test_item.py index ce2b4fd15..daf5d4f59 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -290,7 +290,9 @@ class ItemMetaTest(unittest.TestCase): class ItemMetaClassCellRegression(unittest.TestCase): def test_item_meta_classcell_regression(self): class MyItem(Item, metaclass=ItemMeta): - def __init__(self, *args, **kwargs): + def __init__( + self, *args, **kwargs + ): # pylint: disable=useless-parent-delegation # This call to super() trigger the __classcell__ propagation # requirement. When not done properly raises an error: # TypeError: __class__ set to diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 55ea9eed2..6b4df90d8 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -818,9 +818,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase): ], ) - def test_restrict_xpaths_with_html_entities(self): - super().test_restrict_xpaths_with_html_entities() - @mark.skipif( Version(w3lib_version) < Version("2.0.0"), reason=( diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py index d7f773d5c..99cdf88d9 100644 --- a/tests/test_loader_deprecated.py +++ b/tests/test_loader_deprecated.py @@ -678,11 +678,11 @@ class SelectJmesTestCase(unittest.TestCase): } def test_output(self): - for tl in self.test_list_equals: - expr, test_list, expected = self.test_list_equals[tl] + for k, v in self.test_list_equals.items(): + expr, test_list, expected = v test = SelectJmes(expr)(test_list) self.assertEqual( - test, expected, msg=f'test "{tl}" got {test} expected {expected}' + test, expected, msg=f'test "{k}" got {test} expected {expected}' ) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 3fde5e8c5..9ee248538 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -426,7 +426,7 @@ class SettingsTest(unittest.TestCase): mydict = settings.get("TEST_DICT") self.assertIsInstance(mydict, BaseSettings) self.assertIn("key", mydict) - self.assertEqual(mydict["key"], "val") + self.assertEqual(mydict["key"], "val") # pylint: disable=unsubscriptable-object self.assertEqual(mydict.getpriority("key"), 0) @mock.patch("scrapy.settings.default_settings", default_settings) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 9e5f88f48..be5c6de81 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -353,7 +353,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase): for i, r in enumerate(refs): self.assertIn(r, cache) self.assertEqual(cache[r], i) - del r # delete reference to the last object in the list + del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable # delete half of the objects, make sure that is reflected in the cache for _ in range(max // 2): diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 65b99e0c4..60232f10b 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -75,9 +75,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest): await defer.succeed(42) return "OK" - def test_send_catch_log(self): - return super().test_send_catch_log() - @mark.only_asyncio() class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): @@ -87,9 +84,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest): await asyncio.sleep(0.2) return await get_from_asyncio_queue("OK") - def test_send_catch_log(self): - return super().test_send_catch_log() - class SendCatchLogTest2(unittest.TestCase): def test_error_logged_if_deferred_not_supported(self): diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 460ae40c3..dd1d26448 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -26,7 +26,7 @@ class UtilsSpidersTestCase(unittest.TestCase): self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o]) def test_iter_spider_classes(self): - import tests.test_utils_spider + import tests.test_utils_spider # pylint: disable=import-self it = iter_spider_classes(tests.test_utils_spider) self.assertEqual(set(it), {MySpider1, MySpider2}) diff --git a/tox.ini b/tox.ini index e787c7bf3..4ed9b3bd7 100644 --- a/tox.ini +++ b/tox.ini @@ -53,7 +53,7 @@ commands = basepython = python3 deps = {[testenv:extra-deps]deps} - pylint==3.0.1 + pylint==3.1.0 commands = pylint conftest.py docs extras scrapy setup.py tests