Merge pull request #6253 from wRAR/update-tools

Update tool versions, fix some of the pylint problems

.bandit.yml

@@ -1,20 +1,19 @@
 skips:
-- B101
-- B113  # https://github.com/PyCQA/bandit/issues/1010
-- B105
-- B301
-- B303
-- B307
-- B311
-- B320
-- B321
-- B324
-- B402  # https://github.com/scrapy/scrapy/issues/4180
-- B403
-- B404
-- B406
-- B410
-- B503
-- B603
-- B605
+- B101  # assert_used
+- B105  # hardcoded_password_string
+- B301  # pickle
+- B307  # eval
+- B311  # random
+- B320  # xml_bad_etree
+- B321  # ftplib, https://github.com/scrapy/scrapy/issues/4180
+- B324  # hashlib "Use of weak SHA1 hash for security"
+- B402  # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
+- B403  # import_pickle
+- B404  # import_subprocess
+- B406  # import_xml_sax
+- B410  # import_lxml
+- B411  # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
+- B503  # ssl_with_bad_defaults
+- B603  # subprocess_without_shell_equals_true
+- B605  # start_process_with_a_shell
 exclude_dirs: ['tests']

.flake8

@@ -1,7 +1,7 @@
 [flake8]
 
 max-line-length = 119
-ignore = W503, E203
+ignore = E203, E501, E701, E704, W503
 
 exclude =
     docs/conf.py

.pre-commit-config.yaml

@@ -1,19 +1,19 @@
 repos:
 - repo: https://github.com/PyCQA/bandit
-  rev: 1.7.5
+  rev: 1.7.7
   hooks:
   - id: bandit
     args: [-r, -c, .bandit.yml]
 - repo: https://github.com/PyCQA/flake8
-  rev: 6.1.0
+  rev: 7.0.0
   hooks:
   - id: flake8
 - repo: https://github.com/psf/black.git
-  rev: 23.9.1
+  rev: 24.2.0
   hooks:
   - id: black
 - repo: https://github.com/pycqa/isort
-  rev: 5.12.0
+  rev: 5.13.2
   hooks:
   - id: isort
 - repo: https://github.com/adamchainz/blacken-docs
@@ -21,4 +21,4 @@ repos:
   hooks:
   - id: blacken-docs
     additional_dependencies:
-    - black==23.9.1
+    - black==24.2.0

docs/conf.py

@@ -227,7 +227,7 @@ latex_documents = [
 # A list of regular expressions that match URIs that should not be checked when
 # doing a linkcheck build.
 linkcheck_ignore = [
-    "http://localhost:\d+",
+    r"http://localhost:\d+",
     "http://hg.scrapy.org",
     "http://directory.google.com/",
 ]

docs/topics/addons.rst

@@ -150,8 +150,7 @@ Access the crawler instance:
         def from_crawler(cls, crawler):
             return cls(crawler)
 
-        def update_settings(self, settings):
-            ...
+        def update_settings(self, settings): ...
 
 Use a fallback component:
 

pylintrc

@@ -4,21 +4,14 @@ jobs=1  # >1 hides results
 
 [MESSAGES CONTROL]
 disable=abstract-method,
-        anomalous-backslash-in-string,
         arguments-differ,
         arguments-renamed,
         attribute-defined-outside-init,
         bad-classmethod-argument,
-        bad-mcs-classmethod-argument,
         bare-except,
         broad-except,
         broad-exception-raised,
         c-extension-no-member,
-        catching-non-exception,
-        cell-var-from-loop,
-        comparison-with-callable,
-        consider-using-dict-items,
-        consider-using-in,
         consider-using-with,
         cyclic-import,
         dangerous-default-value,
@@ -32,7 +25,6 @@ disable=abstract-method,
         implicit-str-concat,
         import-error,
         import-outside-toplevel,
-        import-self,
         inconsistent-return-statements,
         inherit-non-class,
         invalid-name,
@@ -44,7 +36,6 @@ disable=abstract-method,
         logging-fstring-interpolation,
         logging-not-lazy,
         lost-exception,
-        method-hidden,
         missing-docstring,
         no-else-raise,
         no-else-return,
@@ -52,7 +43,7 @@ disable=abstract-method,
         no-method-argument,
         no-name-in-module,
         no-self-argument,
-        no-value-for-parameter,
+        no-value-for-parameter,  # https://github.com/pylint-dev/pylint/issues/3268
         not-callable,
         pointless-exception-statement,
         pointless-statement,
@@ -77,14 +68,10 @@ disable=abstract-method,
         too-many-public-methods,
         too-many-return-statements,
         unbalanced-tuple-unpacking,
-        undefined-variable,
-        undefined-loop-variable,
-        unexpected-special-method-signature,
         unnecessary-comprehension,
         unnecessary-dunder-call,
         unnecessary-pass,
         unreachable,
-        unsubscriptable-object,
         unused-argument,
         unused-import,
         unused-private-member,
@@ -92,8 +79,6 @@ disable=abstract-method,
         unused-wildcard-import,
         use-dict-literal,
         used-before-assignment,
-        useless-object-inheritance,  # Required for Python 2 support
         useless-return,
-        useless-super-delegation,
         wildcard-import,
         wrong-import-position

scrapy/commands/__init__.py

@@ -1,6 +1,7 @@
 """
 Base class for Scrapy commands
 """
+
 import argparse
 import os
 from pathlib import Path

scrapy/commands/shell.py

@@ -3,6 +3,7 @@ Scrapy Shell
 
 See documentation in docs/topics/shell.rst
 """
+
 from argparse import Namespace
 from threading import Thread
 from typing import List, Type

scrapy/core/downloader/handlers/__init__.py

@@ -21,9 +21,9 @@ logger = logging.getLogger(__name__)
 class DownloadHandlers:
     def __init__(self, crawler: "Crawler"):
         self._crawler: "Crawler" = crawler
-        self._schemes: Dict[
-            str, Union[str, Callable]
-        ] = {}  # stores acceptable schemes on instancing
+        self._schemes: Dict[str, Union[str, Callable]] = (
+            {}
+        )  # stores acceptable schemes on instancing
         self._handlers: Dict[str, Any] = {}  # stores instanced handlers for schemes
         self._notconfigured: Dict[str, str] = {}  # remembers failed handlers
         handlers: Dict[str, Union[str, Callable]] = without_none_values(

scrapy/core/downloader/handlers/http10.py

@@ -1,5 +1,6 @@
 """Download handlers for http and https schemes
 """
+
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import to_unicode
 

scrapy/core/downloader/middleware.py

@@ -3,6 +3,7 @@ Downloader Middleware manager
 
 See documentation in docs/topics/downloader-middleware.rst
 """
+
 from typing import Any, Callable, Generator, List, Union, cast
 
 from twisted.internet.defer import Deferred, inlineCallbacks

scrapy/core/engine.py

@@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spider.
 For more information see docs/topics/architecture.rst
 
 """
+
 import logging
 from time import time
 from typing import (

scrapy/core/http2/stream.py

@@ -111,17 +111,17 @@ class Stream:
         # Metadata of an HTTP/2 connection stream
         # initialized when stream is instantiated
         self.metadata: Dict = {
-            "request_content_length": 0
-            if self._request.body is None
-            else len(self._request.body),
+            "request_content_length": (
+                0 if self._request.body is None else len(self._request.body)
+            ),
             # Flag to keep track whether the stream has initiated the request
             "request_sent": False,
             # Flag to track whether we have logged about exceeding download warnsize
             "reached_warnsize": False,
             # Each time we send a data frame, we will decrease value by the amount send.
-            "remaining_content_length": 0
-            if self._request.body is None
-            else len(self._request.body),
+            "remaining_content_length": (
+                0 if self._request.body is None else len(self._request.body)
+            ),
             # Flag to keep track whether client (self) have closed this stream
             "stream_closed_local": False,
             # Flag to keep track whether the server has closed the stream

scrapy/core/scraper.py

@@ -1,5 +1,6 @@
 """This module implements the Scraper component which parses responses and
 extracts information from them"""
+
 from __future__ import annotations
 
 import logging

scrapy/core/spidermw.py

@@ -3,6 +3,7 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
+
 import logging
 from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice

scrapy/downloadermiddlewares/defaultheaders.py

@@ -3,6 +3,7 @@ DefaultHeaders downloader middleware
 
 See documentation in docs/topics/downloader-middleware.rst
 """
+
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Iterable, Tuple, Union

scrapy/downloadermiddlewares/downloadtimeout.py

@@ -3,6 +3,7 @@ Download timeout middleware
 
 See documentation in docs/topics/downloader-middleware.rst
 """
+
 from __future__ import annotations
 
 from typing import TYPE_CHECKING, Union

scrapy/downloadermiddlewares/httpcompression.py

@@ -169,7 +169,7 @@ class HttpCompressionMiddleware:
         return to_decode, to_keep
 
     def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:
-        if encoding == b"gzip" or encoding == b"x-gzip":
+        if encoding in {b"gzip", b"x-gzip"}:
             return gunzip(body, max_size=max_size)
         if encoding == b"deflate":
             return _inflate(body, max_size=max_size)

scrapy/downloadermiddlewares/retry.py

@@ -9,6 +9,7 @@ RETRY_HTTP_CODES - which HTTP response codes to retry
 Failed pages are collected on the scraping process and rescheduled at the end,
 once the spider has finished crawling all regular (non failed) pages.
 """
+
 from __future__ import annotations
 
 import warnings

scrapy/exceptions.py

@@ -4,6 +4,7 @@ Scrapy core exceptions
 These exceptions are documented in docs/topics/exceptions.rst. Please don't add
 new exceptions here without documenting them there.
 """
+
 from typing import Any
 
 # Internal

scrapy/extension.py

@@ -3,6 +3,7 @@ The Extension Manager
 
 See documentation in docs/topics/extensions.rst
 """
+
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.conf import build_component_list
 

scrapy/extensions/corestats.py

@@ -1,6 +1,7 @@
 """
 Extension for collecting core stats like items scraped and start/finish times
 """
+
 from datetime import datetime, timezone
 
 from scrapy import signals

scrapy/extensions/memusage.py

@@ -3,6 +3,7 @@ MemoryUsage extension
 
 See documentation in docs/topics/extensions.rst
 """
+
 import logging
 import socket
 import sys

scrapy/extensions/postprocessing.py

@@ -1,6 +1,7 @@
 """
 Extension for processing data before they are exported to feeds.
 """
+
 from bz2 import BZ2File
 from gzip import GzipFile
 from io import IOBase

scrapy/http/headers.py

@@ -113,7 +113,9 @@ class Headers(CaselessDict):
         return ((k, self.getlist(k)) for k in self.keys())
 
     def values(self) -> List[Optional[bytes]]:  # type: ignore[override]
-        return [self[k] for k in self.keys()]
+        return [
+            self[k] for k in self.keys()  # pylint: disable=consider-using-dict-items
+        ]
 
     def to_string(self) -> bytes:
         # cast() can be removed if the headers_dict_to_raw() hint is improved

scrapy/http/request/__init__.py

@@ -4,6 +4,7 @@ requests in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
+
 import inspect
 from typing import (
     Any,
@@ -231,12 +232,16 @@ class Request(object_ref):
         """
         d = {
             "url": self.url,  # urls are safe (safe_string_url)
-            "callback": _find_method(spider, self.callback)
-            if callable(self.callback)
-            else self.callback,
-            "errback": _find_method(spider, self.errback)
-            if callable(self.errback)
-            else self.errback,
+            "callback": (
+                _find_method(spider, self.callback)
+                if callable(self.callback)
+                else self.callback
+            ),
+            "errback": (
+                _find_method(spider, self.errback)
+                if callable(self.errback)
+                else self.errback
+            ),
             "headers": dict(self.headers),
         }
         for attr in self.attributes:

scrapy/http/request/rpc.py

@@ -4,6 +4,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
 
 See documentation in docs/topics/request-response.rst
 """
+
 import xmlrpc.client as xmlrpclib
 from typing import Any, Optional
 

scrapy/http/response/__init__.py

@@ -4,6 +4,7 @@ responses in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
+
 from __future__ import annotations
 
 from ipaddress import IPv4Address, IPv6Address

scrapy/http/response/text.py

@@ -4,6 +4,7 @@ discovering (through HTTP headers) to base Response class.
 
 See documentation in docs/topics/request-response.rst
 """
+
 from __future__ import annotations
 
 import json

scrapy/link.py

@@ -4,6 +4,7 @@ This module defines the Link object used in Link extractors.
 For actual link extractors implementation see scrapy.linkextractors, or
 its documentation in: docs/topics/link-extractors.rst
 """
+
 from typing import Any
 
 

scrapy/linkextractors/__init__.py

@@ -5,6 +5,7 @@ This package contains a collection of Link Extractors.
 
 For more info see docs/topics/link-extractors.rst
 """
+
 import re
 
 # common file extensions that are not followed if they occur in links

scrapy/linkextractors/lxmlhtml.py

@@ -1,6 +1,7 @@
 """
 Link extractor based on lxml.html
 """
+
 import logging
 import operator
 from functools import partial

scrapy/loader/__init__.py

@@ -3,6 +3,7 @@ Item Loader
 
 See documentation in docs/topics/loaders.rst
 """
+
 import itemloaders
 
 from scrapy.item import Item

scrapy/mail.py

@@ -3,6 +3,7 @@ Mail sending helpers
 
 See documentation in docs/topics/email.rst
 """
+
 import logging
 from email import encoders as Encoders
 from email.mime.base import MIMEBase

scrapy/pipelines/__init__.py

@@ -3,6 +3,7 @@ Item pipeline
 
 See documentation in docs/item-pipeline.rst
 """
+
 from typing import Any, List
 
 from twisted.internet.defer import Deferred

scrapy/pipelines/files.py

@@ -3,6 +3,7 @@ Files Pipeline
 
 See documentation in topics/media-pipeline.rst
 """
+
 import base64
 import functools
 import hashlib

scrapy/pipelines/images.py

@@ -3,6 +3,7 @@ Images Pipeline
 
 See documentation in topics/media-pipeline.rst
 """
+
 import functools
 import hashlib
 import warnings

scrapy/responsetypes.py

@@ -2,6 +2,7 @@
 This module implements a class which returns the appropriate Response class
 based on different criteria.
 """
+
 from io import StringIO
 from mimetypes import MimeTypes
 from pkgutil import get_data

scrapy/selector/unified.py

@@ -1,6 +1,7 @@
 """
 XPath selectors based on lxml
 """
+
 from typing import Any, Optional, Type, Union
 
 from parsel import Selector as _ParselSelector

scrapy/settings/__init__.py

@@ -58,7 +58,6 @@ def get_settings_priority(priority: Union[int, str]) -> int:
 
 
 class SettingsAttribute:
-
     """Class for storing data related to settings attributes.
 
     This class is intended for internal usage, you should try Settings class

scrapy/shell.py

@@ -3,6 +3,7 @@
 See documentation in docs/topics/shell.rst
 
 """
+
 import os
 import signal
 

scrapy/spidermiddlewares/httperror.py

@@ -3,6 +3,7 @@ HttpError Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
+
 from __future__ import annotations
 
 import logging

scrapy/spidermiddlewares/offsite.py

@@ -3,6 +3,7 @@ Offsite Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
+
 from __future__ import annotations
 
 import logging

scrapy/spidermiddlewares/referer.py

@@ -2,6 +2,7 @@
 RefererMiddleware: populates Request referer field, based on the Response which
 originated it.
 """
+
 from __future__ import annotations
 
 import warnings

scrapy/spiders/__init__.py

@@ -3,6 +3,7 @@ Base class for Scrapy spiders
 
 See documentation in docs/topics/spiders.rst
 """
+
 from __future__ import annotations
 
 import logging

scrapy/spiders/feed.py

@@ -4,6 +4,7 @@ for scraping from an XML feed.
 
 See documentation in docs/topics/spiders.rst
 """
+
 from scrapy.exceptions import NotConfigured, NotSupported
 from scrapy.selector import Selector
 from scrapy.spiders import Spider

scrapy/statscollectors.py

@@ -1,6 +1,7 @@
 """
 Scrapy extension for collecting scraping stats
 """
+
 import logging
 import pprint
 from typing import TYPE_CHECKING, Any, Dict, Optional

scrapy/utils/defer.py

@@ -1,6 +1,7 @@
 """
 Helper functions for dealing with Twisted deferreds
 """
+
 import asyncio
 import inspect
 from asyncio import Future
@@ -304,13 +305,11 @@ _T = TypeVar("_T")
 
 
 @overload
-def deferred_from_coro(o: _CT) -> Deferred:
-    ...
+def deferred_from_coro(o: _CT) -> Deferred: ...
 
 
 @overload
-def deferred_from_coro(o: _T) -> _T:
-    ...
+def deferred_from_coro(o: _T) -> _T: ...
 
 
 def deferred_from_coro(o: _T) -> Union[Deferred, _T]:

scrapy/utils/deprecate.py

@@ -138,13 +138,11 @@ DEPRECATION_RULES: List[Tuple[str, str]] = []
 
 
 @overload
-def update_classpath(path: str) -> str:
-    ...
+def update_classpath(path: str) -> str: ...
 
 
 @overload
-def update_classpath(path: Any) -> Any:
-    ...
+def update_classpath(path: Any) -> Any: ...
 
 
 def update_classpath(path: Any) -> Any:

scrapy/utils/iterators.py

@@ -225,18 +225,17 @@ def csviter(
 
 
 @overload
-def _body_or_str(obj: Union[Response, str, bytes]) -> str:
-    ...
+def _body_or_str(obj: Union[Response, str, bytes]) -> str: ...
 
 
 @overload
-def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str:
-    ...
+def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[True]) -> str: ...
 
 
 @overload
-def _body_or_str(obj: Union[Response, str, bytes], unicode: Literal[False]) -> bytes:
-    ...
+def _body_or_str(
+    obj: Union[Response, str, bytes], unicode: Literal[False]
+) -> bytes: ...
 
 
 def _body_or_str(

scrapy/utils/misc.py

@@ -1,4 +1,5 @@
 """Helper functions which don't fit anywhere else"""
+
 import ast
 import hashlib
 import inspect

scrapy/utils/ossignal.py

@@ -24,7 +24,11 @@ def install_shutdown_handlers(
     (e.g. Pdb)
     """
     signal.signal(signal.SIGTERM, function)
-    if signal.getsignal(signal.SIGINT) == signal.default_int_handler or override_sigint:
+    if (
+        signal.getsignal(signal.SIGINT)  # pylint: disable=comparison-with-callable
+        == signal.default_int_handler
+        or override_sigint
+    ):
         signal.signal(signal.SIGINT, function)
     # Catch Ctrl-Break in windows
     if hasattr(signal, "SIGBREAK"):

scrapy/utils/python.py

@@ -1,6 +1,7 @@
 """
 This module contains essential stuff that should've come with Python itself ;)
 """
+
 import collections.abc
 import gc
 import inspect
@@ -285,13 +286,11 @@ def equal_attributes(
 
 
 @overload
-def without_none_values(iterable: Mapping) -> dict:
-    ...
+def without_none_values(iterable: Mapping) -> dict: ...
 
 
 @overload
-def without_none_values(iterable: Iterable) -> Iterable:
-    ...
+def without_none_values(iterable: Iterable) -> Iterable: ...
 
 
 def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]:

scrapy/utils/request.py

@@ -44,7 +44,9 @@ def _serialize_headers(
             yield from request.headers.getlist(header)
 
 
-_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
+_fingerprint_cache: (
+    "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
+)
 _fingerprint_cache = WeakKeyDictionary()
 
 
@@ -114,8 +116,7 @@ def fingerprint(
 
 
 class RequestFingerprinterProtocol(Protocol):
-    def fingerprint(self, request: Request) -> bytes:
-        ...
+    def fingerprint(self, request: Request) -> bytes: ...
 
 
 class RequestFingerprinter:

scrapy/utils/response.py

@@ -2,6 +2,7 @@
 This module provides some useful functions for working with
 scrapy.http.Response objects
 """
+
 import os
 import re
 import tempfile
@@ -29,9 +30,9 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:
     return _baseurl_cache[response]
 
 
-_metaref_cache: "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]" = (
-    WeakKeyDictionary()
-)
+_metaref_cache: (
+    "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]"
+) = WeakKeyDictionary()
 
 
 def get_meta_refresh(

scrapy/utils/signal.py

@@ -1,4 +1,5 @@
 """Helper functions for working with signals"""
+
 import collections.abc
 import logging
 from typing import Any as TypingAny
@@ -97,7 +98,10 @@ def send_catch_log_deferred(
             robustApply, receiver, signal=signal, sender=sender, *arguments, **named
         )
         d.addErrback(logerror, receiver)
-        d.addBoth(lambda result: (receiver, result))
+        # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html
+        d.addBoth(
+            lambda result: (receiver, result)  # pylint: disable=cell-var-from-loop
+        )
         dfds.append(d)
     d = DeferredList(dfds)
     d.addCallback(lambda out: [x[1] for x in out])

scrapy/utils/sitemap.py

@@ -4,6 +4,7 @@ Module for processing Sitemaps.
 Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
+
 from typing import Any, Dict, Generator, Iterator, Optional
 from urllib.parse import urljoin
 

scrapy/utils/spider.py

@@ -34,18 +34,15 @@ _T = TypeVar("_T")
 
 # https://stackoverflow.com/questions/60222982
 @overload
-def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator:  # type: ignore[misc]
-    ...
+def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ...  # type: ignore[overload-overlap]
 
 
 @overload
-def iterate_spider_output(result: CoroutineType) -> Deferred:
-    ...
+def iterate_spider_output(result: CoroutineType) -> Deferred: ...
 
 
 @overload
-def iterate_spider_output(result: _T) -> Iterable:
-    ...
+def iterate_spider_output(result: _T) -> Iterable: ...
 
 
 def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:
@@ -83,8 +80,7 @@ def spidercls_for_request(
     default_spidercls: Type[Spider],
     log_none: bool = ...,
     log_multiple: bool = ...,
-) -> Type[Spider]:
-    ...
+) -> Type[Spider]: ...
 
 
 @overload
@@ -94,8 +90,7 @@ def spidercls_for_request(
     default_spidercls: Literal[None],
     log_none: bool = ...,
     log_multiple: bool = ...,
-) -> Optional[Type[Spider]]:
-    ...
+) -> Optional[Type[Spider]]: ...
 
 
 @overload
@@ -105,8 +100,7 @@ def spidercls_for_request(
     *,
     log_none: bool = ...,
     log_multiple: bool = ...,
-) -> Optional[Type[Spider]]:
-    ...
+) -> Optional[Type[Spider]]: ...
 
 
 def spidercls_for_request(

scrapy/utils/ssl.py

@@ -1,6 +1,6 @@
 from typing import Any, Optional
 
-import OpenSSL._util as pyOpenSSLutil  # type: ignore[import-untyped]
+import OpenSSL._util as pyOpenSSLutil
 import OpenSSL.SSL
 import OpenSSL.version
 from OpenSSL.crypto import X509Name

scrapy/utils/url.py

@@ -5,6 +5,7 @@ library.
 Some of the functions that used to be imported from this module have been moved
 to the w3lib.url module. Always import those from there instead.
 """
+
 import re
 from typing import TYPE_CHECKING, Iterable, Optional, Type, Union, cast
 from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse

tests/mocks/dummydbm.py

@@ -1,4 +1,5 @@
 """DBM-like dummy module"""
+
 import collections
 from typing import Any, DefaultDict
 

tests/spiders.py

@@ -1,6 +1,7 @@
 """
 Some spiders used for testing and benchmarking
 """
+
 import asyncio
 import time
 from urllib.parse import urlencode

tests/test_commands.py

@@ -991,38 +991,11 @@ class MySpider(scrapy.Spider):
 class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
     spider_filename = "myspider.pyw"
 
-    def setUp(self):
-        super().setUp()
-
     def test_start_requests_errors(self):
         log = self.get_log(self.badspider, name="badspider.pyw")
         self.assertIn("start_requests", log)
         self.assertIn("badspider.pyw", log)
 
-    def test_run_good_spider(self):
-        super().test_run_good_spider()
-
-    def test_runspider(self):
-        super().test_runspider()
-
-    def test_runspider_dnscache_disabled(self):
-        super().test_runspider_dnscache_disabled()
-
-    def test_runspider_log_level(self):
-        super().test_runspider_log_level()
-
-    def test_runspider_log_short_names(self):
-        super().test_runspider_log_short_names()
-
-    def test_runspider_no_spider_found(self):
-        super().test_runspider_no_spider_found()
-
-    def test_output(self):
-        super().test_output()
-
-    def test_overwrite_output(self):
-        super().test_overwrite_output()
-
     def test_runspider_unable_to_load(self):
         raise unittest.SkipTest("Already Tested in 'RunSpiderCommandTest' ")
 

tests/test_exporters.py

@@ -121,7 +121,9 @@ class BaseItemExporterTest(unittest.TestCase):
         self.assertEqual(name, "John\xa3")
 
         ie = self._get_exporter(fields_to_export={"name": "名稱"})
-        self.assertEqual(list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")])
+        self.assertEqual(
+            list(ie._get_serialized_fields(self.i)), [("名稱", "John\xa3")]
+        )
 
     def test_field_custom_serializer(self):
         i = self.custom_field_item_class(name="John\xa3", age="22")

tests/test_item.py

@@ -290,7 +290,9 @@ class ItemMetaTest(unittest.TestCase):
 class ItemMetaClassCellRegression(unittest.TestCase):
     def test_item_meta_classcell_regression(self):
         class MyItem(Item, metaclass=ItemMeta):
-            def __init__(self, *args, **kwargs):
+            def __init__(
+                self, *args, **kwargs
+            ):  # pylint: disable=useless-parent-delegation
                 # This call to super() trigger the __classcell__ propagation
                 # requirement. When not done properly raises an error:
                 # TypeError: __class__ set to <class '__main__.MyItem'>

tests/test_linkextractors.py

@@ -818,9 +818,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
             ],
         )
 
-    def test_restrict_xpaths_with_html_entities(self):
-        super().test_restrict_xpaths_with_html_entities()
-
     @mark.skipif(
         Version(w3lib_version) < Version("2.0.0"),
         reason=(

tests/test_loader_deprecated.py

@@ -678,11 +678,11 @@ class SelectJmesTestCase(unittest.TestCase):
     }
 
     def test_output(self):
-        for tl in self.test_list_equals:
-            expr, test_list, expected = self.test_list_equals[tl]
+        for k, v in self.test_list_equals.items():
+            expr, test_list, expected = v
             test = SelectJmes(expr)(test_list)
             self.assertEqual(
-                test, expected, msg=f'test "{tl}" got {test} expected {expected}'
+                test, expected, msg=f'test "{k}" got {test} expected {expected}'
             )
 
 

tests/test_pipeline_media.py

@@ -22,9 +22,9 @@ from scrapy.utils.test import get_crawler
 try:
     from PIL import Image  # noqa: imported just to check for the import error
 except ImportError:
-    skip_pillow: Optional[
-        str
-    ] = "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow"
+    skip_pillow: Optional[str] = (
+        "Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow"
+    )
 else:
     skip_pillow = None
 

tests/test_responsetypes.py

@@ -33,7 +33,10 @@ class ResponseTypesTest(unittest.TestCase):
             ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),
             ("attachment;filename=data高.doc".encode("gbk"), Response),
             ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),
-            ("attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"), XmlResponse),
+            (
+                "attachment;filename=日本語版Wikipedia.xml".encode("iso2022_jp"),
+                XmlResponse,
+            ),
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_disposition(source)

tests/test_settings/__init__.py

@@ -426,7 +426,7 @@ class SettingsTest(unittest.TestCase):
         mydict = settings.get("TEST_DICT")
         self.assertIsInstance(mydict, BaseSettings)
         self.assertIn("key", mydict)
-        self.assertEqual(mydict["key"], "val")
+        self.assertEqual(mydict["key"], "val")  # pylint: disable=unsubscriptable-object
         self.assertEqual(mydict.getpriority("key"), 0)
 
     @mock.patch("scrapy.settings.default_settings", default_settings)

tests/test_utils_datatypes.py

@@ -353,7 +353,7 @@ class LocalWeakReferencedCacheTest(unittest.TestCase):
         for i, r in enumerate(refs):
             self.assertIn(r, cache)
             self.assertEqual(cache[r], i)
-        del r  # delete reference to the last object in the list
+        del r  # delete reference to the last object in the list  # pylint: disable=undefined-loop-variable
 
         # delete half of the objects, make sure that is reflected in the cache
         for _ in range(max // 2):

tests/test_utils_signal.py

@@ -75,9 +75,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
         await defer.succeed(42)
         return "OK"
 
-    def test_send_catch_log(self):
-        return super().test_send_catch_log()
-
 
 @mark.only_asyncio()
 class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
@@ -87,9 +84,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
         await asyncio.sleep(0.2)
         return await get_from_asyncio_queue("OK")
 
-    def test_send_catch_log(self):
-        return super().test_send_catch_log()
-
 
 class SendCatchLogTest2(unittest.TestCase):
     def test_error_logged_if_deferred_not_supported(self):

tests/test_utils_spider.py

@@ -26,7 +26,7 @@ class UtilsSpidersTestCase(unittest.TestCase):
         self.assertEqual(list(iterate_spider_output([r, i, o])), [r, i, o])
 
     def test_iter_spider_classes(self):
-        import tests.test_utils_spider
+        import tests.test_utils_spider  # pylint: disable=import-self
 
         it = iter_spider_classes(tests.test_utils_spider)
         self.assertEqual(set(it), {MySpider1, MySpider2})

tests/test_webclient.py

@@ -2,6 +2,7 @@
 from twisted.internet import defer
 Tests borrowed from the twisted.web.client tests.
 """
+
 import shutil
 from pathlib import Path
 from tempfile import mkdtemp

tox.ini

@@ -29,14 +29,14 @@ install_command =
 [testenv:typing]
 basepython = python3
 deps =
-    mypy==1.6.1
-    typing-extensions==4.8.0
+    mypy==1.8.0
+    typing-extensions==4.10.0
     types-attrs==19.1.0
-    types-lxml==2023.10.21
-    types-Pillow==10.1.0.0
-    types-Pygments==2.16.0.0
-    types-pyOpenSSL==23.3.0.0
-    types-setuptools==68.2.0.0
+    types-lxml==2024.2.9
+    types-Pillow==10.2.0.20240213
+    types-Pygments==2.17.0.20240106
+    types-pyOpenSSL==24.0.0.20240130
+    types-setuptools==69.1.0.20240223
     # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211
     w3lib >= 2.1.2
 commands =
@@ -53,7 +53,7 @@ commands =
 basepython = python3
 deps =
     {[testenv:extra-deps]deps}
-    pylint==3.0.1
+    pylint==3.1.0
 commands =
     pylint conftest.py docs extras scrapy setup.py tests