Bump tool versions. (#6941)

.pre-commit-config.yaml

@@ -1,8 +1,8 @@
 repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.9.3
+  rev: v0.12.2
   hooks:
-    - id: ruff
+    - id: ruff-check
       args: [ --fix ]
     - id: ruff-format
 - repo: https://github.com/adamchainz/blacken-docs
@@ -10,7 +10,7 @@ repos:
   hooks:
   - id: blacken-docs
     additional_dependencies:
-    - black==24.10.0
+    - black==25.1.0
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v5.0.0
   hooks:

conftest.py

@@ -85,7 +85,7 @@ def requires_uvloop(request):
     if not request.node.get_closest_marker("requires_uvloop"):
         return
     try:
-        import uvloop
+        import uvloop  # noqa: PLC0415
 
         del uvloop
     except ImportError:
@@ -97,7 +97,7 @@ def requires_botocore(request):
     if not request.node.get_closest_marker("requires_botocore"):
         return
     try:
-        import botocore
+        import botocore  # noqa: PLC0415
 
         del botocore
     except ImportError:
@@ -109,7 +109,7 @@ def requires_boto3(request):
     if not request.node.get_closest_marker("requires_boto3"):
         return
     try:
-        import boto3
+        import boto3  # noqa: PLC0415
 
         del boto3
     except ImportError:

pyproject.toml

@@ -106,10 +106,6 @@ follow_imports = "skip"
 module = "scrapy.settings.default_settings"
 ignore_errors = true
 
-[[tool.mypy.overrides]]
-module = "itemadapter"
-implicit_reexport = true
-
 [[tool.mypy.overrides]]
 module = "twisted"
 implicit_reexport = true
@@ -394,6 +390,9 @@ banned-module-level-imports = [
     "twisted.internet.reactor",
 ]
 
+[tool.ruff.lint.isort]
+split-on-trailing-comma = false
+
 [tool.ruff.lint.per-file-ignores]
 # Circular import workarounds
 "scrapy/linkextractors/__init__.py" = ["E402"]

scrapy/__init__.py

@@ -31,11 +31,11 @@ version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."
 
 def __getattr__(name: str):
     if name == "twisted_version":
-        import warnings  # pylint: disable=reimported
+        import warnings  # noqa: PLC0415  # pylint: disable=reimported
 
-        from twisted import version as _txv
+        from twisted import version as _txv  # noqa: PLC0415
 
-        from scrapy.exceptions import ScrapyDeprecationWarning
+        from scrapy.exceptions import ScrapyDeprecationWarning  # noqa: PLC0415
 
         warnings.warn(
             "The scrapy.twisted_version attribute is deprecated, use twisted.version instead",

scrapy/commands/bench.py

@@ -10,6 +10,7 @@ import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Response, TextResponse
 from scrapy.linkextractors import LinkExtractor
+from scrapy.utils.test import get_testenv
 
 if TYPE_CHECKING:
     import argparse
@@ -35,8 +36,6 @@ class Command(ScrapyCommand):
 
 class _BenchServer:
     def __enter__(self) -> None:
-        from scrapy.utils.test import get_testenv
-
         pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
         self.proc = subprocess.Popen(  # noqa: S603
             pargs, stdout=subprocess.PIPE, env=get_testenv()

scrapy/commands/genspider.py

@@ -206,7 +206,7 @@ class Command(ScrapyCommand):
 
         # a file with the same name exists in the target directory
         spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
-        spiders_dir = Path(cast(str, spiders_module.__file__)).parent
+        spiders_dir = Path(cast("str", spiders_module.__file__)).parent
         spiders_dir_abs = spiders_dir.resolve()
         path = spiders_dir_abs / (name + ".py")
         if path.exists():

scrapy/contracts/__init__.py

@@ -52,7 +52,7 @@ class Contract:
                 cb_result = cb(response, **cb_kwargs)
                 if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
                     raise TypeError("Contracts don't support async callbacks")
-                return list(cast(Iterable[Any], iterate_spider_output(cb_result)))
+                return list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
 
             request.callback = wrapper
 
@@ -68,7 +68,7 @@ class Contract:
                 cb_result = cb(response, **cb_kwargs)
                 if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
                     raise TypeError("Contracts don't support async callbacks")
-                output = list(cast(Iterable[Any], iterate_spider_output(cb_result)))
+                output = list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
                 try:
                     results.startTest(self.testcase_post)
                     self.post_process(output)
@@ -181,7 +181,7 @@ class ContractsManager:
         def cb_wrapper(response: Response, **cb_kwargs: Any) -> None:
             try:
                 output = cb(response, **cb_kwargs)
-                output = list(cast(Iterable[Any], iterate_spider_output(output)))
+                output = list(cast("Iterable[Any]", iterate_spider_output(output)))
             except Exception:
                 case = _create_testcase(method, "callback")
                 results.addError(case, sys.exc_info())

scrapy/core/downloader/__init__.py

@@ -162,7 +162,7 @@ class Downloader:
 
     def get_slot_key(self, request: Request) -> str:
         if self.DOWNLOAD_SLOT in request.meta:
-            return cast(str, request.meta[self.DOWNLOAD_SLOT])
+            return cast("str", request.meta[self.DOWNLOAD_SLOT])
 
         key = urlparse_cached(request).hostname or ""
         if self.ip_concurrency:

scrapy/core/downloader/handlers/http11.py

@@ -386,6 +386,7 @@ class ScrapyAgent:
             if not proxy_port:
                 proxy_port = 443 if proxy_parsed.scheme == "https" else 80
             if urlparse_cached(request).scheme == "https":
+                assert proxy_host is not None
                 proxyAuth = request.headers.get(b"Proxy-Authorization", None)
                 proxyConf = (proxy_host, proxy_port, proxyAuth)
                 return self._TunnelingAgent(
@@ -430,7 +431,7 @@ class ScrapyAgent:
             method,
             to_bytes(url, encoding="ascii"),
             headers,
-            cast(IBodyProducer, bodyproducer),
+            cast("IBodyProducer", bodyproducer),
         )
         # set download latency
         d.addCallback(self._cb_latency, request, start_time)

scrapy/core/downloader/handlers/s3.py

@@ -51,8 +51,8 @@ class S3DownloadHandler:
         self.anon = kw.get("anon")
 
         self._signer = None
-        import botocore.auth
+        import botocore.auth  # noqa: PLC0415
-        import botocore.credentials
+        import botocore.credentials  # noqa: PLC0415
 
         kw.pop("anon", None)
         if kw:
@@ -87,7 +87,7 @@ class S3DownloadHandler:
         if self.anon:
             request = request.replace(url=url)
         else:
-            import botocore.awsrequest
+            import botocore.awsrequest  # noqa: PLC0415
 
             awsrequest = botocore.awsrequest.AWSRequest(
                 method=request.method,

scrapy/core/downloader/middleware.py

@@ -6,7 +6,6 @@ See documentation in docs/topics/downloader-middleware.rst
 
 from __future__ import annotations
 
-from collections.abc import Callable
 from typing import TYPE_CHECKING, Any, cast
 
 from twisted.internet.defer import Deferred, inlineCallbacks
@@ -18,7 +17,7 @@ from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import _defer_sleep, deferred_from_coro
 
 if TYPE_CHECKING:
-    from collections.abc import Generator
+    from collections.abc import Callable, Generator
 
     from scrapy import Spider
     from scrapy.settings import BaseSettings
@@ -51,7 +50,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             request: Request,
         ) -> Generator[Deferred[Any], Any, Response | Request]:
             for method in self.methods["process_request"]:
-                method = cast(Callable, method)
+                method = cast("Callable", method)
                 response = yield deferred_from_coro(
                     method(request=request, spider=spider)
                 )
@@ -76,7 +75,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 return response
 
             for method in self.methods["process_response"]:
-                method = cast(Callable, method)
+                method = cast("Callable", method)
                 response = yield deferred_from_coro(
                     method(request=request, response=response, spider=spider)
                 )
@@ -94,7 +93,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             exception: Exception,
         ) -> Generator[Deferred[Any], Any, Response | Request]:
             for method in self.methods["process_exception"]:
-                method = cast(Callable, method)
+                method = cast("Callable", method)
                 response = yield deferred_from_coro(
                     method(request=request, exception=exception, spider=spider)
                 )

scrapy/core/engine.py

@@ -17,13 +17,11 @@ from twisted.internet.defer import CancelledError, Deferred, inlineCallbacks, su
 from twisted.python.failure import Failure
 
 from scrapy import signals
+from scrapy.core.scheduler import BaseScheduler
 from scrapy.core.scraper import Scraper
 from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.utils.asyncio import (
+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
-    AsyncioLoopingCall,
-    create_looping_call,
-)
 from scrapy.utils.defer import (
     deferred_f_from_coro_f,
     deferred_from_coro,
@@ -39,7 +37,6 @@ if TYPE_CHECKING:
     from twisted.internet.task import LoopingCall
 
     from scrapy.core.downloader import Downloader
-    from scrapy.core.scheduler import BaseScheduler
     from scrapy.crawler import Crawler
     from scrapy.logformatter import LogFormatter
     from scrapy.settings import BaseSettings, Settings
@@ -123,8 +120,6 @@ class ExecutionEngine:
             raise
 
     def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]:
-        from scrapy.core.scheduler import BaseScheduler
-
         scheduler_cls: type[BaseScheduler] = load_object(settings["SCHEDULER"])
         if not issubclass(scheduler_cls, BaseScheduler):
             raise TypeError(
@@ -506,7 +501,7 @@ class ExecutionEngine:
         dfd.addErrback(log_failure("Scraper close failure"))
 
         if hasattr(self._slot.scheduler, "close"):
-            dfd.addBoth(lambda _: cast(_Slot, self._slot).scheduler.close(reason))
+            dfd.addBoth(lambda _: cast("_Slot", self._slot).scheduler.close(reason))
             dfd.addErrback(log_failure("Scheduler close failure"))
 
         dfd.addBoth(

scrapy/core/scheduler.py

@@ -491,7 +491,7 @@ class Scheduler(BaseScheduler):
         if not path.exists():
             return []
         with path.open(encoding="utf-8") as f:
-            return cast(list[int], json.load(f))
+            return cast("list[int]", json.load(f))
 
     def _write_dqs_state(self, dqdir: str, state: list[int]) -> None:
         with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:

scrapy/core/spidermw.py

@@ -138,7 +138,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         spider: Spider,
     ) -> Deferred[Iterable[_T] | AsyncIterator[_T]]:
         for method in self.methods["process_spider_input"]:
-            method = cast(Callable, method)
+            method = cast("Callable", method)
             try:
                 result = method(response=response, spider=spider)
                 if result is not None:
@@ -166,7 +166,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 yield from iterable
             except Exception as ex:
                 exception_result = cast(
-                    Union[Failure, MutableChain[_T]],
+                    "Union[Failure, MutableChain[_T]]",
                     self._process_spider_exception(
                         response, spider, ex, exception_processor_index
                     ),
@@ -182,7 +182,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                     yield r
             except Exception as ex:
                 exception_result = cast(
-                    Union[Failure, MutableAsyncChain[_T]],
+                    "Union[Failure, MutableAsyncChain[_T]]",
                     self._process_spider_exception(
                         response, spider, ex, exception_processor_index
                     ),
@@ -212,7 +212,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         for method_index, method in enumerate(method_list, start=start_index):
             if method is None:
                 continue
-            method = cast(Callable, method)
+            method = cast("Callable", method)
             result = method(response=response, exception=exception, spider=spider)
             if _isiterable(result):
                 # stop exception handling by handing control over to the
@@ -227,7 +227,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 if dfd.called:
                     # the result is available immediately if _process_spider_output didn't do downgrading
                     return cast(
-                        Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result
+                        "Union[MutableChain[_T], MutableAsyncChain[_T]]", dfd.result
                     )
                 # we forbid waiting here because otherwise we would need to return a deferred from
                 # _process_spider_exception too, which complicates the architecture

scrapy/crawler.py

@@ -8,11 +8,7 @@ import signal
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any, TypeVar
 
-from twisted.internet.defer import (
+from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
-    Deferred,
-    DeferredList,
-    inlineCallbacks,
-)
 
 from scrapy import Spider, signals
 from scrapy.addons import AddonManager

scrapy/downloadermiddlewares/redirect.py

@@ -202,6 +202,6 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
         redirected = self._redirect_request_using_get(request, url)
         if urlparse_cached(redirected).scheme not in {"http", "https"}:
             return response
-        if cast(float, interval) < self._maxdelay:
+        if cast("float", interval) < self._maxdelay:
             return self._redirect(redirected, request, spider, "meta refresh")
         return response

scrapy/extensions/feedexport.py

@@ -218,7 +218,7 @@ class S3FeedStorage(BlockingFeedStorage):
         region_name: str | None = None,
     ):
         try:
-            import boto3.session
+            import boto3.session  # noqa: PLC0415
         except ImportError:
             raise NotConfigured("missing boto3 library")
         u = urlparse(uri)
@@ -317,7 +317,7 @@ class GCSFeedStorage(BlockingFeedStorage):
 
     def _store_in_thread(self, file: IO[bytes]) -> None:
         file.seek(0)
-        from google.cloud.storage import Client
+        from google.cloud.storage import Client  # noqa: PLC0415
 
         client = Client(project=self.project_id)
         bucket = client.get_bucket(self.bucket_name)
@@ -413,7 +413,7 @@ class FeedSlot:
             self.file = self.storage.open(self.spider)
             if "postprocessing" in self.feed_options:
                 self.file = cast(
-                    IO[bytes],
+                    "IO[bytes]",
                     PostProcessingManager(
                         self.feed_options["postprocessing"],
                         self.file,
@@ -662,7 +662,7 @@ class FeedExporter:
 
     def _load_components(self, setting_prefix: str) -> dict[str, Any]:
         conf = without_none_values(
-            cast(dict[str, str], self.settings.getwithbase(setting_prefix))
+            cast("dict[str, str]", self.settings.getwithbase(setting_prefix))
         )
         d = {}
         for k, v in conf.items():

scrapy/extensions/httpcache.py

@@ -307,7 +307,7 @@ class DbmCacheStorage:
         if 0 < self.expiration_secs < time() - float(ts):
             return None  # expired
 
-        return cast(dict[str, Any], pickle.loads(db[f"{key}_data"]))  # noqa: S301
+        return cast("dict[str, Any]", pickle.loads(db[f"{key}_data"]))  # noqa: S301
 
 
 class FilesystemCacheStorage:
@@ -389,7 +389,7 @@ class FilesystemCacheStorage:
         if 0 < self.expiration_secs < time() - mtime:
             return None  # expired
         with self._open(metapath, "rb") as f:
-            return cast(dict[str, Any], pickle.load(f))  # noqa: S301
+            return cast("dict[str, Any]", pickle.load(f))  # noqa: S301
 
 
 def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]:

scrapy/extensions/logstats.py

@@ -5,10 +5,7 @@ from typing import TYPE_CHECKING
 
 from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured
-from scrapy.utils.asyncio import (
+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
-    AsyncioLoopingCall,
-    create_looping_call,
-)
 
 if TYPE_CHECKING:
     from twisted.internet.task import LoopingCall

scrapy/extensions/memusage.py

@@ -16,10 +16,7 @@ from typing import TYPE_CHECKING
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 from scrapy.mail import MailSender
-from scrapy.utils.asyncio import (
+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
-    AsyncioLoopingCall,
-    create_looping_call,
-)
 from scrapy.utils.engine import get_engine_status
 
 if TYPE_CHECKING:

scrapy/extensions/periodic_log.py

@@ -6,10 +6,7 @@ from typing import TYPE_CHECKING, Any
 
 from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured
-from scrapy.utils.asyncio import (
+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
-    AsyncioLoopingCall,
-    create_looping_call,
-)
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
 if TYPE_CHECKING:

scrapy/extensions/postprocessing.py

@@ -142,7 +142,7 @@ class PostProcessingManager(IOBase):
         :return: returns number of bytes written
         :rtype: int
         """
-        return cast(int, self.head_plugin.write(data))
+        return cast("int", self.head_plugin.write(data))
 
     def tell(self) -> int:
         return self.file.tell()

scrapy/extensions/telnet.py

@@ -12,6 +12,8 @@ import os
 import pprint
 from typing import TYPE_CHECKING, Any
 
+from twisted.conch import manhole, telnet
+from twisted.conch.insults import insults
 from twisted.internet import protocol
 
 from scrapy import signals
@@ -22,7 +24,6 @@ from scrapy.utils.reactor import listen_tcp
 from scrapy.utils.trackref import print_live_refs
 
 if TYPE_CHECKING:
-    from twisted.conch import telnet
     from twisted.internet.tcp import Port
 
     # typing.Self requires Python 3.11
@@ -76,10 +77,6 @@ class TelnetConsole(protocol.ServerFactory):
         self.port.stopListening()
 
     def protocol(self) -> telnet.TelnetTransport:
-        # these import twisted.internet.reactor
-        from twisted.conch import manhole, telnet
-        from twisted.conch.insults import insults
-
         class Portal:
             """An implementation of IPortal"""
 

scrapy/http/cookies.py

@@ -161,7 +161,7 @@ class WrappedRequest:
         HTML document, and the user had no option to approve the automatic
         fetching of the image, this should be true.
         """
-        return cast(bool, self.request.meta.get("is_unverifiable", False))
+        return cast("bool", self.request.meta.get("is_unverifiable", False))
 
     @property
     def full_url(self) -> str:
@@ -181,7 +181,7 @@ class WrappedRequest:
 
     @property
     def origin_req_host(self) -> str:
-        return cast(str, urlparse_cached(self.request).hostname)
+        return cast("str", urlparse_cached(self.request).hostname)
 
     def has_header(self, name: str) -> bool:
         return name in self.request.headers

scrapy/http/headers.py

@@ -69,19 +69,19 @@ class Headers(CaselessDict):
 
     def __getitem__(self, key: AnyStr) -> bytes | None:
         try:
-            return cast(list[bytes], super().__getitem__(key))[-1]
+            return cast("list[bytes]", super().__getitem__(key))[-1]
         except IndexError:
             return None
 
     def get(self, key: AnyStr, def_val: Any = None) -> bytes | None:
         try:
-            return cast(list[bytes], super().get(key, def_val))[-1]
+            return cast("list[bytes]", super().get(key, def_val))[-1]
         except IndexError:
             return None
 
     def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]:
         try:
-            return cast(list[bytes], super().__getitem__(key))
+            return cast("list[bytes]", super().__getitem__(key))
         except KeyError:
             if def_val is not None:
                 return self.normvalue(def_val)

scrapy/http/request/form.py

@@ -11,13 +11,7 @@ from collections.abc import Iterable
 from typing import TYPE_CHECKING, Any, Optional, Union, cast
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
 
-from lxml.html import (
+from parsel.csstranslator import HTMLTranslator
-    FormElement,
-    InputElement,
-    MultipleSelectOptions,
-    SelectElement,
-    TextareaElement,
-)
 from w3lib.html import strip_html5_whitespace
 
 from scrapy.http.request import Request
@@ -25,6 +19,13 @@ from scrapy.utils.python import is_listlike, to_bytes
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
+    from lxml.html import (
+        FormElement,
+        InputElement,
+        MultipleSelectOptions,
+        SelectElement,
+        TextareaElement,
+    )
     from typing_extensions import Self
 
     from scrapy.http.response.text import TextResponse
@@ -76,8 +77,6 @@ class FormRequest(Request):
         kwargs.setdefault("encoding", response.encoding)
 
         if formcss is not None:
-            from parsel.csstranslator import HTMLTranslator
-
             formxpath = HTMLTranslator().css_to_xpath(formcss)
 
         form = _get_form(response, formname, formid, formnumber, formxpath)
@@ -107,7 +106,7 @@ def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str:
     values = [
         (to_bytes(k, enc), to_bytes(v, enc))
         for k, vs in seq
-        for v in (cast(Iterable[str], vs) if is_listlike(vs) else [cast(str, vs)])
+        for v in (cast("Iterable[str]", vs) if is_listlike(vs) else [cast("str", vs)])
     ]
     return urlencode(values, doseq=True)
 
@@ -128,12 +127,12 @@ def _get_form(
     if formname is not None:
         f = root.xpath(f'//form[@name="{formname}"]')
         if f:
-            return cast(FormElement, f[0])
+            return cast("FormElement", f[0])
 
     if formid is not None:
         f = root.xpath(f'//form[@id="{formid}"]')
         if f:
-            return cast(FormElement, f[0])
+            return cast("FormElement", f[0])
 
     # Get form element from xpath, if not found, go up
     if formxpath is not None:
@@ -142,7 +141,7 @@ def _get_form(
             el = nodes[0]
             while True:
                 if el.tag == "form":
-                    return cast(FormElement, el)
+                    return cast("FormElement", el)
                 el = el.getparent()
                 if el is None:
                     break
@@ -153,7 +152,7 @@ def _get_form(
         form = forms[formnumber]
     except IndexError:
         raise IndexError(f"Form number {formnumber} not found in {response}")
-    return cast(FormElement, form)
+    return cast("FormElement", form)
 
 
 def _get_inputs(
@@ -201,7 +200,7 @@ def _value(
     n = ele.name
     v = ele.value
     if ele.tag == "select":
-        return _select_value(cast(SelectElement, ele), n, v)
+        return _select_value(cast("SelectElement", ele), n, v)
     return n, v
 
 
@@ -251,7 +250,7 @@ def _get_clickable(
         except IndexError:
             pass
         else:
-            return (el.get("name"), el.get("value") or "")
+            return (cast("str", el.get("name")), el.get("value") or "")
 
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such

scrapy/http/response/text.py

@@ -104,13 +104,14 @@ class TextResponse(Response):
 
     @memoizemethod_noargs
     def _headers_encoding(self) -> str | None:
-        content_type = cast(bytes, self.headers.get(b"Content-Type", b""))
+        content_type = cast("bytes", self.headers.get(b"Content-Type", b""))
         return http_content_type_encoding(to_unicode(content_type, encoding="latin-1"))
 
     def _body_inferred_encoding(self) -> str:
         if self._cached_benc is None:
             content_type = to_unicode(
-                cast(bytes, self.headers.get(b"Content-Type", b"")), encoding="latin-1"
+                cast("bytes", self.headers.get(b"Content-Type", b"")),
+                encoding="latin-1",
             )
             benc, ubody = html_to_unicode(
                 content_type,
@@ -141,31 +142,25 @@ class TextResponse(Response):
 
     @property
     def selector(self) -> Selector:
-        from scrapy.selector import Selector
+        # circular import
+        from scrapy.selector import Selector  # noqa: PLC0415
 
         if self._cached_selector is None:
             self._cached_selector = Selector(self)
         return self._cached_selector
 
     def jmespath(self, query: str, **kwargs: Any) -> SelectorList:
-        from scrapy.selector import SelectorList
-
         if not hasattr(self.selector, "jmespath"):
             raise AttributeError(
                 "Please install parsel >= 1.8.1 to get jmespath support"
             )
-
+        return cast("SelectorList", self.selector.jmespath(query, **kwargs))
-        return cast(SelectorList, self.selector.jmespath(query, **kwargs))
 
     def xpath(self, query: str, **kwargs: Any) -> SelectorList:
-        from scrapy.selector import SelectorList
+        return cast("SelectorList", self.selector.xpath(query, **kwargs))
-
-        return cast(SelectorList, self.selector.xpath(query, **kwargs))
 
     def css(self, query: str) -> SelectorList:
-        from scrapy.selector import SelectorList
+        return cast("SelectorList", self.selector.css(query))
-
-        return cast(SelectorList, self.selector.css(query))
 
     def follow(
         self,

scrapy/linkextractors/lxmlhtml.py

@@ -71,12 +71,12 @@ class LxmlParserLinkExtractor:
         self.scan_tag: Callable[[str], bool] = (
             tag
             if callable(tag)
-            else cast(Callable[[str], bool], partial(operator.eq, tag))
+            else cast("Callable[[str], bool]", partial(operator.eq, tag))
         )
         self.scan_attr: Callable[[str], bool] = (
             attr
             if callable(attr)
-            else cast(Callable[[str], bool], partial(operator.eq, attr))
+            else cast("Callable[[str], bool]", partial(operator.eq, attr))
         )
         self.process_attr: Callable[[Any], Any] = (
             process if callable(process) else _identity
@@ -84,7 +84,7 @@ class LxmlParserLinkExtractor:
         self.unique: bool = unique
         self.strip: bool = strip
         self.link_key: Callable[[Link], str] = (
-            cast(Callable[[Link], str], operator.attrgetter("url"))
+            cast("Callable[[Link], str]", operator.attrgetter("url"))
             if canonicalized
             else _canonicalize_link_url
         )

scrapy/mail.py

@@ -223,7 +223,8 @@ class MailSender:
     def _create_sender_factory(
         self, to_addrs: list[str], msg: IO[bytes], d: Deferred[Any]
     ) -> ESMTPSenderFactory:
-        from twisted.mail.smtp import ESMTPSenderFactory
+        # imports twisted.internet.reactor
+        from twisted.mail.smtp import ESMTPSenderFactory  # noqa: PLC0415
 
         factory_keywords: dict[str, Any] = {
             "heloFallback": True,

scrapy/middleware.py

@@ -63,7 +63,7 @@ class MiddlewareManager(ABC):
             method_name = "__new__"
         if instance is None:
             raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
-        return cast(_T, instance)
+        return cast("_T", instance)
 
     @classmethod
     def from_settings(cls, settings: Settings, crawler: Crawler | None = None) -> Self:

scrapy/pipelines/files.py

@@ -169,7 +169,7 @@ class S3FilesStore:
     def __init__(self, uri: str):
         if not is_botocore_available():
             raise NotConfigured("missing botocore library")
-        import botocore.session
+        import botocore.session  # noqa: PLC0415
 
         session = botocore.session.get_session()
         self.s3_client = session.create_client(
@@ -284,7 +284,7 @@ class GCSFilesStore:
     POLICY = None
 
     def __init__(self, uri: str):
-        from google.cloud import storage
+        from google.cloud import storage  # noqa: PLC0415
 
         client = storage.Client(project=self.GCS_PROJECT_ID)
         bucket, prefix = uri[5:].split("/", 1)
@@ -317,7 +317,7 @@ class GCSFilesStore:
 
         blob_path = self._get_blob_path(path)
         return cast(
-            Deferred[StatInfo],
+            "Deferred[StatInfo]",
             deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess),
         )
 
@@ -531,7 +531,9 @@ class FilesPipeline(MediaPipeline):
 
     @classmethod
     def _update_stores(cls, settings: BaseSettings) -> None:
-        s3store: type[S3FilesStore] = cast(type[S3FilesStore], cls.STORE_SCHEMES["s3"])
+        s3store: type[S3FilesStore] = cast(
+            "type[S3FilesStore]", cls.STORE_SCHEMES["s3"]
+        )
         s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]
         s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]
         s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"]
@@ -542,13 +544,13 @@ class FilesPipeline(MediaPipeline):
         s3store.POLICY = settings["FILES_STORE_S3_ACL"]
 
         gcs_store: type[GCSFilesStore] = cast(
-            type[GCSFilesStore], cls.STORE_SCHEMES["gs"]
+            "type[GCSFilesStore]", cls.STORE_SCHEMES["gs"]
         )
         gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"]
         gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None
 
         ftp_store: type[FTPFilesStore] = cast(
-            type[FTPFilesStore], cls.STORE_SCHEMES["ftp"]
+            "type[FTPFilesStore]", cls.STORE_SCHEMES["ftp"]
         )
         ftp_store.FTP_USERNAME = settings["FTP_USER"]
         ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"]
@@ -742,5 +744,5 @@ class FilesPipeline(MediaPipeline):
             media_ext = ""
             media_type = mimetypes.guess_type(request.url)[0]
             if media_type:
-                media_ext = cast(str, mimetypes.guess_extension(media_type))
+                media_ext = cast("str", mimetypes.guess_extension(media_type))
         return f"full/{media_guid}{media_ext}"

scrapy/pipelines/images.py

@@ -63,7 +63,7 @@ class ImagesPipeline(FilesPipeline):
         crawler: Crawler | None = None,
     ):
         try:
-            from PIL import Image
+            from PIL import Image  # noqa: PLC0415
 
             self._Image = Image
         except ImportError:

scrapy/pqueues.py

@@ -4,7 +4,6 @@ import hashlib
 import logging
 from typing import TYPE_CHECKING, Protocol, cast
 
-from scrapy import Request
 from scrapy.utils.misc import build_from_crawler
 
 if TYPE_CHECKING:
@@ -13,6 +12,7 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Request
     from scrapy.core.downloader import Downloader
     from scrapy.crawler import Crawler
 
@@ -211,7 +211,7 @@ class ScrapyPriorityQueue:
         except KeyError:
             queue = self.queues[self.curprio]
         # Protocols can't declare optional members
-        return cast(Request, queue.peek())  # type: ignore[attr-defined]
+        return cast("Request", queue.peek())  # type: ignore[attr-defined]
 
     def close(self) -> list[int]:
         active: set[int] = set()

scrapy/robotstxt.py

@@ -4,6 +4,9 @@ import logging
 import sys
 from abc import ABCMeta, abstractmethod
 from typing import TYPE_CHECKING
+from urllib.robotparser import RobotFileParser
+
+from protego import Protego
 
 from scrapy.utils.python import to_unicode
 
@@ -67,8 +70,6 @@ class RobotParser(metaclass=ABCMeta):
 
 class PythonRobotParser(RobotParser):
     def __init__(self, robotstxt_body: bytes, spider: Spider | None):
-        from urllib.robotparser import RobotFileParser
-
         self.spider: Spider | None = spider
         body_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
         self.rp: RobotFileParser = RobotFileParser()
@@ -87,7 +88,7 @@ class PythonRobotParser(RobotParser):
 
 class RerpRobotParser(RobotParser):
     def __init__(self, robotstxt_body: bytes, spider: Spider | None):
-        from robotexclusionrulesparser import RobotExclusionRulesParser
+        from robotexclusionrulesparser import RobotExclusionRulesParser  # noqa: PLC0415
 
         self.spider: Spider | None = spider
         self.rp: RobotExclusionRulesParser = RobotExclusionRulesParser()
@@ -107,8 +108,6 @@ class RerpRobotParser(RobotParser):
 
 class ProtegoRobotParser(RobotParser):
     def __init__(self, robotstxt_body: bytes, spider: Spider | None):
-        from protego import Protego
-
         self.spider: Spider | None = spider
         body_decoded = decode_robotstxt(robotstxt_body, spider)
         self.rp = Protego.parse(body_decoded)

scrapy/settings/__init__.py

@@ -336,7 +336,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         stored.
         """
         if len(self) > 0:
-            return max(cast(int, self.getpriority(name)) for name in self)
+            return max(cast("int", self.getpriority(name)) for name in self)
         return get_settings_priority("default")
 
     def replace_in_component_priority_dict(
@@ -519,11 +519,11 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         """
         self._assert_mutability()
         if isinstance(values, str):
-            values = cast(dict[_SettingsKeyT, Any], json.loads(values))
+            values = cast("dict[_SettingsKeyT, Any]", json.loads(values))
         if values is not None:
             if isinstance(values, BaseSettings):
                 for name, value in values.items():
-                    self.set(name, value, cast(int, values.getpriority(name)))
+                    self.set(name, value, cast("int", values.getpriority(name)))
             else:
                 for name, value in values.items():
                     self.set(name, value, priority)
@@ -533,7 +533,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
             raise KeyError(name)
         self._assert_mutability()
         priority = get_settings_priority(priority)
-        if priority >= cast(int, self.getpriority(name)):
+        if priority >= cast("int", self.getpriority(name)):
             del self.attributes[name]
 
     def __delitem__(self, name: _SettingsKeyT) -> None:

scrapy/settings/default_settings.py

@@ -535,9 +535,9 @@ WARN_ON_GENERATOR_RETURN_VALUE = True
 
 def __getattr__(name: str):
     if name == "CONCURRENT_REQUESTS_PER_IP":
-        import warnings
+        import warnings  # noqa: PLC0415
 
-        from scrapy.exceptions import ScrapyDeprecationWarning
+        from scrapy.exceptions import ScrapyDeprecationWarning  # noqa: PLC0415
 
         warnings.warn(
             "The scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_IP attribute is deprecated, use scrapy.settings.default_settings.CONCURRENT_REQUESTS_PER_DOMAIN instead.",

scrapy/shell.py

@@ -16,6 +16,7 @@ from twisted.internet import defer, threads
 from twisted.python import threadable
 from w3lib.url import any_to_uri
 
+import scrapy
 from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest
 from scrapy.http import Request, Response
@@ -164,8 +165,6 @@ class Shell:
         request: Request | None = None,
         spider: Spider | None = None,
     ) -> None:
-        import scrapy
-
         self.vars["scrapy"] = scrapy
         self.vars["crawler"] = self.crawler
         self.vars["item"] = self.item_class()

scrapy/spidermiddlewares/referer.py

@@ -315,7 +315,7 @@ def _load_policy_class(
     from https://www.w3.org/TR/referrer-policy/#referrer-policies
     """
     try:
-        return cast(type[ReferrerPolicy], load_object(policy))
+        return cast("type[ReferrerPolicy]", load_object(policy))
     except ValueError:
         tokens = [token.strip() for token in policy.lower().split(",")]
         # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header

scrapy/spiders/__init__.py

@@ -55,7 +55,8 @@ class Spider(object_ref):
 
     @property
     def logger(self) -> SpiderLoggerAdapter:
-        from scrapy.utils.log import SpiderLoggerAdapter
+        # circular import
+        from scrapy.utils.log import SpiderLoggerAdapter  # noqa: PLC0415
 
         logger = logging.getLogger(self.name)
         return SpiderLoggerAdapter(logger, {"spider": self})

scrapy/spiders/crawl.py

@@ -12,8 +12,6 @@ import warnings
 from collections.abc import AsyncIterator, Awaitable, Callable
 from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast
 
-from twisted.python.failure import Failure
-
 from scrapy.http import HtmlResponse, Request, Response
 from scrapy.link import Link
 from scrapy.linkextractors import LinkExtractor
@@ -26,6 +24,8 @@ from scrapy.utils.spider import iterate_spider_output
 if TYPE_CHECKING:
     from collections.abc import Iterable, Sequence
 
+    from twisted.python.failure import Failure
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
@@ -81,12 +81,14 @@ class Rule:
     def _compile(self, spider: Spider) -> None:
         # this replaces method names with methods and we can't express this in type hints
         self.callback = cast("CallbackT", _get_method(self.callback, spider))
-        self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider))
+        self.errback = cast(
+            "Callable[[Failure], Any]", _get_method(self.errback, spider)
+        )
         self.process_links = cast(
-            ProcessLinksT, _get_method(self.process_links, spider)
+            "ProcessLinksT", _get_method(self.process_links, spider)
         )
         self.process_request = cast(
-            ProcessRequestT, _get_method(self.process_request, spider)
+            "ProcessRequestT", _get_method(self.process_request, spider)
         )
 
 
@@ -141,13 +143,13 @@ class CrawlSpider(Spider):
                 for lnk in rule.link_extractor.extract_links(response)
                 if lnk not in seen
             ]
-            for link in cast(ProcessLinksT, rule.process_links)(links):
+            for link in cast("ProcessLinksT", rule.process_links)(links):
                 seen.add(link)
                 request = self._build_request(rule_index, link)
-                yield cast(ProcessRequestT, rule.process_request)(request, response)
+                yield cast("ProcessRequestT", rule.process_request)(request, response)
 
     def _callback(self, response: Response, **cb_kwargs: Any) -> Any:
-        rule = self._rules[cast(int, response.meta["rule"])]
+        rule = self._rules[cast("int", response.meta["rule"])]
         return self.parse_with_rules(
             response,
             cast("CallbackT", rule.callback),
@@ -156,9 +158,9 @@ class CrawlSpider(Spider):
         )
 
     def _errback(self, failure: Failure) -> Iterable[Any]:
-        rule = self._rules[cast(int, failure.request.meta["rule"])]  # type: ignore[attr-defined]
+        rule = self._rules[cast("int", failure.request.meta["rule"])]  # type: ignore[attr-defined]
         return self._handle_failure(
-            failure, cast(Callable[[Failure], Any], rule.errback)
+            failure, cast("Callable[[Failure], Any]", rule.errback)
         )
 
     async def parse_with_rules(

scrapy/spiders/init.py

@@ -1,15 +1,16 @@
 from __future__ import annotations
 
 import warnings
-from collections.abc import AsyncIterator, Iterable
 from typing import TYPE_CHECKING, Any, cast
 
-from scrapy import Request
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.spiders import Spider
 from scrapy.utils.spider import iterate_spider_output
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Iterable
+
+    from scrapy import Request
     from scrapy.http import Response
 
 
@@ -39,7 +40,7 @@ class InitSpider(Spider):
 
     def start_requests(self) -> Iterable[Request]:
         self._postinit_reqs: Iterable[Request] = super().start_requests()
-        return cast(Iterable[Request], iterate_spider_output(self.init_request()))
+        return cast("Iterable[Request]", iterate_spider_output(self.init_request()))
 
     def initialized(self, response: Response | None = None) -> Any:
         """This method must be set as the callback of your last initialization

scrapy/utils/boto.py

@@ -3,7 +3,7 @@
 
 def is_botocore_available() -> bool:
     try:
-        import botocore  # noqa: F401
+        import botocore  # noqa: F401,PLC0415
 
         return True
     except ImportError:

scrapy/utils/conf.py

@@ -153,7 +153,7 @@ def feed_process_params_from_cli(
     suitable to be used as the FEEDS setting.
     """
     valid_output_formats: Iterable[str] = without_none_values(
-        cast(dict[str, str], settings.getwithbase("FEED_EXPORTERS"))
+        cast("dict[str, str]", settings.getwithbase("FEED_EXPORTERS"))
     ).keys()
 
     def check_valid_format(output_format: str) -> None:

scrapy/utils/console.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import code
 from collections.abc import Callable
 from functools import wraps
 from typing import TYPE_CHECKING, Any
@@ -16,13 +17,13 @@ def _embed_ipython_shell(
 ) -> EmbedFuncT:
     """Start an IPython Shell"""
     try:
-        from IPython.terminal.embed import InteractiveShellEmbed  # noqa: T100
+        from IPython.terminal.embed import InteractiveShellEmbed  # noqa: T100,PLC0415
-        from IPython.terminal.ipapp import load_default_config
+        from IPython.terminal.ipapp import load_default_config  # noqa: PLC0415
     except ImportError:
-        from IPython.frontend.terminal.embed import (  # type: ignore[no-redef]  # noqa: T100
+        from IPython.frontend.terminal.embed import (  # type: ignore[no-redef]  # noqa: T100,PLC0415
             InteractiveShellEmbed,
         )
-        from IPython.frontend.terminal.ipapp import (  # type: ignore[no-redef]
+        from IPython.frontend.terminal.ipapp import (  # type: ignore[no-redef]  # noqa: PLC0415
             load_default_config,
         )
 
@@ -46,7 +47,7 @@ def _embed_bpython_shell(
     namespace: dict[str, Any] = {}, banner: str = ""
 ) -> EmbedFuncT:
     """Start a bpython shell"""
-    import bpython
+    import bpython  # noqa: PLC0415
 
     @wraps(_embed_bpython_shell)
     def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
@@ -59,7 +60,7 @@ def _embed_ptpython_shell(
     namespace: dict[str, Any] = {}, banner: str = ""
 ) -> EmbedFuncT:
     """Start a ptpython shell"""
-    import ptpython.repl  # pylint: disable=import-error
+    import ptpython.repl  # noqa: PLC0415  # pylint: disable=import-error
 
     @wraps(_embed_ptpython_shell)
     def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
@@ -73,14 +74,12 @@ def _embed_standard_shell(
     namespace: dict[str, Any] = {}, banner: str = ""
 ) -> EmbedFuncT:
     """Start a standard python shell"""
-    import code
-
     try:  # readline module is only available on unix systems
-        import readline
+        import readline  # noqa: PLC0415
     except ImportError:
         pass
     else:
-        import rlcompleter  # noqa: F401
+        import rlcompleter  # noqa: F401,PLC0415
 
         readline.parse_and_bind("tab:complete")  # type: ignore[attr-defined]
 

scrapy/utils/datatypes.py

@@ -32,7 +32,8 @@ class CaselessDict(dict):
     __slots__ = ()
 
     def __new__(cls, *args: Any, **kwargs: Any) -> Self:
-        from scrapy.http.headers import Headers
+        # circular import
+        from scrapy.http.headers import Headers  # noqa: PLC0415
 
         if issubclass(cls, CaselessDict) and not issubclass(cls, Headers):
             warnings.warn(

scrapy/utils/defer.py

@@ -12,12 +12,7 @@ from collections.abc import Awaitable, Coroutine, Iterable, Iterator
 from functools import wraps
 from typing import TYPE_CHECKING, Any, Generic, TypeVar, cast, overload
 
-from twisted.internet.defer import (
+from twisted.internet.defer import Deferred, DeferredList, fail, succeed
-    Deferred,
-    DeferredList,
-    fail,
-    succeed,
-)
 from twisted.internet.task import Cooperator
 from twisted.python import failure
 
@@ -414,7 +409,7 @@ def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2:
         if not is_asyncio_available():
             # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines
             # that use asyncio, e.g. "await asyncio.sleep(1)"
-            return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o))
+            return Deferred.fromCoroutine(cast("Coroutine[Deferred[Any], Any, _T]", o))
         # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor
         return Deferred.fromFuture(asyncio.ensure_future(o))
     return o

scrapy/utils/display.py

@@ -34,11 +34,11 @@ def _colorize(text: str, colorize: bool = True) -> str:
     if not colorize or not sys.stdout.isatty() or not _tty_supports_color():
         return text
     try:
-        from pygments import highlight
+        from pygments import highlight  # noqa: PLC0415
     except ImportError:
         return text
-    from pygments.formatters import TerminalFormatter
+    from pygments.formatters import TerminalFormatter  # noqa: PLC0415
-    from pygments.lexers import PythonLexer
+    from pygments.lexers import PythonLexer  # noqa: PLC0415
 
     return highlight(text, PythonLexer(), TerminalFormatter())
 

scrapy/utils/iterators.py

@@ -147,12 +147,12 @@ class _StreamReader:
     def _read_string(self, n: int = 65535) -> bytes:
         s, e = self._ptr, self._ptr + n
         self._ptr = e
-        return cast(bytes, self._text)[s:e]
+        return cast("bytes", self._text)[s:e]
 
     def _read_unicode(self, n: int = 65535) -> bytes:
         s, e = self._ptr, self._ptr + n
         self._ptr = e
-        return cast(str, self._text)[s:e].encode("utf-8")
+        return cast("str", self._text)[s:e].encode("utf-8")
 
 
 def csviter(

scrapy/utils/log.py

@@ -5,9 +5,9 @@ import pprint
 import sys
 from collections.abc import MutableMapping
 from logging.config import dictConfig
-from types import TracebackType
 from typing import TYPE_CHECKING, Any, Optional, cast
 
+from twisted.internet import asyncioreactor
 from twisted.python import log as twisted_log
 from twisted.python.failure import Failure
 
@@ -16,6 +16,8 @@ from scrapy.settings import Settings, _SettingsKeyT
 from scrapy.utils.versions import get_versions
 
 if TYPE_CHECKING:
+    from types import TracebackType
+
     from scrapy.crawler import Crawler
     from scrapy.logformatter import LogFormatterResult
 
@@ -33,7 +35,7 @@ def failure_to_exc_info(
         return (
             failure.type,
             failure.value,
-            cast(Optional[TracebackType], failure.getTracebackObject()),
+            cast("Optional[TracebackType]", failure.getTracebackObject()),
         )
     return None
 
@@ -182,7 +184,7 @@ def log_scrapy_info(settings: Settings) -> None:
 
 
 def log_reactor_info() -> None:
-    from twisted.internet import asyncioreactor, reactor
+    from twisted.internet import reactor
 
     logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
     if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor):
@@ -240,7 +242,7 @@ def logformatter_adapter(
     message = logkws.get("msg") or ""
     # NOTE: This also handles 'args' being an empty dict, that case doesn't
     # play well in logger.log calls
-    args = cast(dict[str, Any], logkws) if not logkws.get("args") else logkws["args"]
+    args = cast("dict[str, Any]", logkws) if not logkws.get("args") else logkws["args"]
 
     return (level, message, args)
 

scrapy/utils/misc.py

@@ -9,7 +9,6 @@ import os
 import re
 import warnings
 from collections import deque
-from collections.abc import Iterable
 from contextlib import contextmanager
 from functools import partial
 from importlib import import_module
@@ -21,7 +20,7 @@ from scrapy.item import Item
 from scrapy.utils.datatypes import LocalWeakReferencedCache
 
 if TYPE_CHECKING:
-    from collections.abc import Callable, Iterator
+    from collections.abc import Callable, Iterable, Iterator
     from types import ModuleType
 
     from scrapy import Spider
@@ -41,7 +40,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]:
     if arg is None:
         return []
     if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):
-        return cast(Iterable[Any], arg)
+        return cast("Iterable[Any]", arg)
     return [arg]
 
 
@@ -202,7 +201,7 @@ def build_from_crawler(
         method_name = "__new__"
     if instance is None:
         raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
-    return cast(T, instance)
+    return cast("T", instance)
 
 
 @contextmanager

scrapy/utils/reactor.py

@@ -29,7 +29,7 @@ if TYPE_CHECKING:
 _T = TypeVar("_T")
 
 
-def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:  # type: ignore[return]  # pylint: disable=inconsistent-return-statements
+def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:  # type: ignore[return]  # pylint: disable=inconsistent-return-statements  # noqa: RET503
     """Like reactor.listenTCP but tries different ports in a range."""
     from twisted.internet import reactor
 
@@ -39,7 +39,7 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:
         return reactor.listenTCP(0, factory, interface=host)
     if len(portrange) == 1:
         return reactor.listenTCP(portrange[0], factory, interface=host)
-    for x in range(portrange[0], portrange[1] + 1):  # noqa: RET503
+    for x in range(portrange[0], portrange[1] + 1):
         try:
             return reactor.listenTCP(x, factory, interface=host)
         except error.CannotListenError:
@@ -60,7 +60,8 @@ class CallLaterOnce(Generic[_T]):
         self._deferreds: list[Deferred] = []
 
     def schedule(self, delay: float = 0) -> None:
-        from scrapy.utils.asyncio import call_later
+        # circular import
+        from scrapy.utils.asyncio import call_later  # noqa: PLC0415
 
         if self._call is None:
             self._call = call_later(delay, self)
@@ -70,7 +71,8 @@ class CallLaterOnce(Generic[_T]):
             self._call.cancel()
 
     def __call__(self) -> _T:
-        from scrapy.utils.asyncio import call_later
+        # circular import
+        from scrapy.utils.asyncio import call_later  # noqa: PLC0415
 
         self._call = None
         result = self._func(*self._a, **self._kw)
@@ -82,7 +84,8 @@ class CallLaterOnce(Generic[_T]):
         return result
 
     async def wait(self):
-        from scrapy.utils.defer import maybe_deferred_to_future
+        # circular import
+        from scrapy.utils.defer import maybe_deferred_to_future  # noqa: PLC0415
 
         d = Deferred()
         self._deferreds.append(d)

scrapy/utils/response.py

@@ -91,7 +91,8 @@ def open_in_browser(
             if "item name" not in response.body:
                 open_in_browser(response)
     """
-    from scrapy.http import HtmlResponse, TextResponse
+    # circular imports
+    from scrapy.http import HtmlResponse, TextResponse  # noqa: PLC0415
 
     # XXX: this implementation is a bit dirty and could be improved
     body = response.body

scrapy/utils/sitemap.py

@@ -24,7 +24,7 @@ class Sitemap:
         xmlp = lxml.etree.XMLParser(
             recover=True, remove_comments=True, resolve_entities=False
         )
-        self._root = lxml.etree.fromstring(xmltext, parser=xmlp)  # noqa: S320
+        self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
         rt = self._root.tag
         assert isinstance(rt, str)
         self.type = rt.split("}", 1)[1] if "}" in rt else rt

scrapy/utils/test.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 import asyncio
 import os
 import warnings
+from ftplib import FTP
 from importlib import import_module
 from pathlib import Path
 from posixpath import split
@@ -14,7 +15,9 @@ from typing import TYPE_CHECKING, Any, TypeVar, cast
 from unittest import TestCase, mock
 
 from twisted.trial.unittest import SkipTest
+from twisted.web.client import Agent
 
+from scrapy.crawler import CrawlerRunner
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.deprecate import create_deprecated_class
@@ -59,7 +62,7 @@ def skip_if_no_boto() -> None:
 def get_gcs_content_and_delete(
     bucket: Any, path: str
 ) -> tuple[bytes, list[dict[str, str]], Any]:
-    from google.cloud import storage
+    from google.cloud import storage  # noqa: PLC0415
 
     warnings.warn(
         "The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",
@@ -83,8 +86,6 @@ def get_ftp_content_and_delete(
     password: str,
     use_active_mode: bool = False,
 ) -> bytes:
-    from ftplib import FTP
-
     warnings.warn(
         "The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",
         category=ScrapyDeprecationWarning,
@@ -137,8 +138,6 @@ def get_crawler(
     will be used to populate the crawler settings with a project level
     priority.
     """
-    from scrapy.crawler import CrawlerRunner
-
     # When needed, useful settings can be added here, e.g. ones that prevent
     # deprecation warnings.
     settings: dict[str, Any] = {
@@ -192,7 +191,7 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
     """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
     classes and set their proper return values.
     """
-    from google.cloud.storage import Blob, Bucket, Client
+    from google.cloud.storage import Blob, Bucket, Client  # noqa: PLC0415
 
     warnings.warn(
         "The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.",
@@ -213,7 +212,6 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
 
 def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:
     from twisted.internet import reactor
-    from twisted.web.client import Agent  # imports twisted.internet.reactor
 
     agent = Agent(reactor)
     return cast("Deferred[TxResponse]", agent.request(b"GET", url.encode("utf-8")))

scrapy/utils/testproc.py

@@ -6,7 +6,6 @@ import warnings
 from typing import TYPE_CHECKING, cast
 
 from twisted.internet.defer import Deferred
-from twisted.internet.error import ProcessTerminated
 from twisted.internet.protocol import ProcessProtocol
 
 from scrapy.exceptions import ScrapyDeprecationWarning
@@ -14,6 +13,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 if TYPE_CHECKING:
     from collections.abc import Iterable
 
+    from twisted.internet.error import ProcessTerminated
     from twisted.python.failure import Failure
 
 
@@ -55,7 +55,7 @@ class ProcessTest:
             msg += "\n"
             msg += f"\n>>> stderr <<<\n{pp.err.decode()}"
             raise RuntimeError(msg)
-        return cast(int, pp.exitcode), pp.out, pp.err
+        return cast("int", pp.exitcode), pp.out, pp.err
 
 
 class TestProcessProtocol(ProcessProtocol):
@@ -72,5 +72,5 @@ class TestProcessProtocol(ProcessProtocol):
         self.err += data
 
     def processEnded(self, status: Failure) -> None:
-        self.exitcode = cast(ProcessTerminated, status.value).exitCode
+        self.exitcode = cast("ProcessTerminated", status.value).exitCode
         self.deferred.callback(self)

tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py

@@ -21,7 +21,7 @@ def log_task_exception(task: Task) -> None:
     try:
         task.result()
     except Exception:
-        logging.exception("Crawl task failed")
+        logging.exception("Crawl task failed")  # noqa: LOG015
 
 
 process = AsyncCrawlerProcess()

tests/test_command_genspider.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 import os
 from pathlib import Path
 
+import pytest
+
 from tests.test_commands import TestCommandBase, TestProjectBase
 
 
@@ -15,7 +17,16 @@ class TestGenspiderCommand(TestCommandBase):
         assert self.call("genspider", "test_name", "test.com") == 0
         assert Path(self.proj_mod_path, "spiders", "test_name.py").exists()
 
-    def test_template(self, tplname="crawl"):
+    @pytest.mark.parametrize(
+        "tplname",
+        [
+            "basic",
+            "crawl",
+            "xmlfeed",
+            "csvfeed",
+        ],
+    )
+    def test_template(self, tplname: str) -> None:
         args = [f"--template={tplname}"] if tplname else []
         spname = "test_spider"
         spmodule = f"{self.project_name}.spiders.{spname}"
@@ -35,15 +46,6 @@ class TestGenspiderCommand(TestCommandBase):
         )
         assert modify_time_after == modify_time_before
 
-    def test_template_basic(self):
-        self.test_template("basic")
-
-    def test_template_csvfeed(self):
-        self.test_template("csvfeed")
-
-    def test_template_xmlfeed(self):
-        self.test_template("xmlfeed")
-
     def test_list(self):
         assert self.call("genspider", "--list") == 0
 
@@ -57,7 +59,8 @@ class TestGenspiderCommand(TestCommandBase):
             self.proj_mod_path, "spiders", f"{self.project_name}.py"
         ).exists()
 
-    def test_same_filename_as_existing_spider(self, force=False):
+    @pytest.mark.parametrize("force", [True, False])
+    def test_same_filename_as_existing_spider(self, force: bool) -> None:
         file_name = "example"
         file_path = Path(self.proj_mod_path, "spiders", f"{file_name}.py")
         assert self.call("genspider", file_name, "example.com") == 0
@@ -90,83 +93,60 @@ class TestGenspiderCommand(TestCommandBase):
             file_contents_after = file_path.read_text(encoding="utf-8")
             assert file_contents_after == file_contents_before
 
-    def test_same_filename_as_existing_spider_force(self):
+    @pytest.mark.parametrize(
-        self.test_same_filename_as_existing_spider(force=True)
+        ("url", "domain"),
-
+        [
-    def test_url(self, url="test.com", domain="test.com"):
+            ("test.com", "test.com"),
+            ("https://test.com", "test.com"),
+        ],
+    )
+    def test_url(self, url: str, domain: str) -> None:
         assert self.call("genspider", "--force", "test_name", url) == 0
-        assert (
+        m = self.find_in_file(
-            self.find_in_file(
+            self.proj_mod_path / "spiders" / "test_name.py",
-                Path(self.proj_mod_path, "spiders", "test_name.py"),
+            r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]",
-                r"allowed_domains\s*=\s*\[['\"](.+)['\"]\]",
-            ).group(1)
-            == domain
         )
-        assert (
+        assert m is not None
-            self.find_in_file(
+        assert m.group(1) == domain
-                Path(self.proj_mod_path, "spiders", "test_name.py"),
+        m = self.find_in_file(
-                r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
+            self.proj_mod_path / "spiders" / "test_name.py",
-            ).group(1)
+            r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
-            == f"https://{domain}"
         )
+        assert m is not None
+        assert m.group(1) == f"https://{domain}"
 
-    def test_url_schema(self):
+    @pytest.mark.parametrize(
-        self.test_url("https://test.com", "test.com")
+        ("url", "expected", "template"),
-
+        [
-    def test_template_start_urls(
+            # basic
-        self, url="test.com", expected="https://test.com", template="basic"
+            ("https://test.com", "https://test.com", "basic"),
-    ):
+            ("http://test.com", "http://test.com", "basic"),
+            ("http://test.com/other/path", "http://test.com/other/path", "basic"),
+            ("test.com/other/path", "https://test.com/other/path", "basic"),
+            # crawl
+            ("https://test.com", "https://test.com", "crawl"),
+            ("http://test.com", "http://test.com", "crawl"),
+            ("http://test.com/other/path", "http://test.com/other/path", "crawl"),
+            ("test.com/other/path", "https://test.com/other/path", "crawl"),
+            ("test.com", "https://test.com", "crawl"),
+            # xmlfeed
+            ("https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"),
+            ("http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"),
+            ("test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"),
+            # csvfeed
+            ("https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"),
+            ("http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"),
+            ("test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"),
+        ],
+    )
+    def test_template_start_urls(self, url: str, expected: str, template: str) -> None:
         assert self.call("genspider", "-t", template, "--force", "test_name", url) == 0
-        assert (
+        m = self.find_in_file(
-            self.find_in_file(
+            self.proj_mod_path / "spiders" / "test_name.py",
-                Path(self.proj_mod_path, "spiders", "test_name.py"),
+            r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
-                r"start_urls\s*=\s*\[['\"](.+)['\"]\]",
-            ).group(1)
-            == expected
-        )
-
-    def test_genspider_basic_start_urls(self):
-        self.test_template_start_urls("https://test.com", "https://test.com", "basic")
-        self.test_template_start_urls("http://test.com", "http://test.com", "basic")
-        self.test_template_start_urls(
-            "http://test.com/other/path", "http://test.com/other/path", "basic"
-        )
-        self.test_template_start_urls(
-            "test.com/other/path", "https://test.com/other/path", "basic"
-        )
-
-    def test_genspider_crawl_start_urls(self):
-        self.test_template_start_urls("https://test.com", "https://test.com", "crawl")
-        self.test_template_start_urls("http://test.com", "http://test.com", "crawl")
-        self.test_template_start_urls(
-            "http://test.com/other/path", "http://test.com/other/path", "crawl"
-        )
-        self.test_template_start_urls(
-            "test.com/other/path", "https://test.com/other/path", "crawl"
-        )
-        self.test_template_start_urls("test.com", "https://test.com", "crawl")
-
-    def test_genspider_xmlfeed_start_urls(self):
-        self.test_template_start_urls(
-            "https://test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"
-        )
-        self.test_template_start_urls(
-            "http://test.com/feed.xml", "http://test.com/feed.xml", "xmlfeed"
-        )
-        self.test_template_start_urls(
-            "test.com/feed.xml", "https://test.com/feed.xml", "xmlfeed"
-        )
-
-    def test_genspider_csvfeed_start_urls(self):
-        self.test_template_start_urls(
-            "https://test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"
-        )
-        self.test_template_start_urls(
-            "http://test.com/feed.xml", "http://test.com/feed.xml", "csvfeed"
-        )
-        self.test_template_start_urls(
-            "test.com/feed.csv", "https://test.com/feed.csv", "csvfeed"
         )
+        assert m is not None
+        assert m.group(1) == expected
 
 
 class TestGenspiderStandaloneCommand(TestProjectBase):
@@ -174,7 +154,8 @@ class TestGenspiderStandaloneCommand(TestProjectBase):
         self.call("genspider", "example", "example.com")
         assert Path(self.temp_path, "example.py").exists()
 
-    def test_same_name_as_existing_file(self, force=False):
+    @pytest.mark.parametrize("force", [True, False])
+    def test_same_name_as_existing_file(self, force: bool) -> None:
         file_name = "example"
         file_path = Path(self.temp_path, file_name + ".py")
         p, out, err = self.proc("genspider", file_name, "example.com")
@@ -203,6 +184,3 @@ class TestGenspiderStandaloneCommand(TestProjectBase):
             assert modify_time_after == modify_time_before
             file_contents_after = file_path.read_text(encoding="utf-8")
             assert file_contents_after == file_contents_before
-
-    def test_same_name_as_existing_file_force(self):
-        self.test_same_name_as_existing_file(force=True)

tests/test_command_runspider.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import asyncio
 import inspect
 import platform
 import sys
@@ -204,8 +205,6 @@ class MySpider(scrapy.Spider):
                 "TWISTED_REACTOR=twisted.internet.asyncioreactor.AsyncioSelectorReactor",
             ],
         )
-        import asyncio
-
         if sys.platform != "win32":
             loop = asyncio.new_event_loop()
         else:

tests/test_commands.py

@@ -9,7 +9,7 @@ from io import StringIO
 from pathlib import Path
 from shutil import rmtree
 from tempfile import TemporaryFile, mkdtemp
-from typing import TYPE_CHECKING, Any
+from typing import Any
 from unittest import mock
 
 import pytest
@@ -22,9 +22,6 @@ from scrapy.utils.python import to_unicode
 from scrapy.utils.reactor import _asyncio_reactor_path
 from scrapy.utils.test import get_testenv
 
-if TYPE_CHECKING:
-    import os
-
 
 class EmptyCommand(ScrapyCommand):
     def short_desc(self) -> str:
@@ -110,10 +107,11 @@ class TestProjectBase:
 
         return p, to_unicode(stdout), to_unicode(stderr)
 
-    def find_in_file(self, filename: str | os.PathLike, regex) -> re.Match | None:
+    @staticmethod
+    def find_in_file(filename: Path, regex: str) -> re.Match | None:
         """Find first pattern occurrence in file"""
         pattern = re.compile(regex)
-        with Path(filename).open("r", encoding="utf-8") as f:
+        with filename.open("r", encoding="utf-8") as f:
             for line in f:
                 match = pattern.search(line)
                 if match is not None:

tests/test_core_downloader.py

@@ -4,7 +4,7 @@ import shutil
 import warnings
 from pathlib import Path
 from tempfile import mkdtemp
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, cast
 
 import OpenSSL.SSL
 import pytest
@@ -14,7 +14,6 @@ from twisted.trial import unittest
 from twisted.web import server, static
 from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
 from twisted.web.client import Response as TxResponse
-from twisted.web.iweb import IBodyProducer
 
 from scrapy.core.downloader import Slot
 from scrapy.core.downloader.contextfactory import (
@@ -29,6 +28,9 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler
 from tests.mockserver import PayloadResource, ssl_context_factory
 
+if TYPE_CHECKING:
+    from twisted.web.iweb import IBodyProducer
+
 
 class TestSlot:
     def test_repr(self):
@@ -78,12 +80,12 @@ class TestContextFactoryBase(unittest.TestCase):
         agent = Agent(reactor, contextFactory=client_context_factory)
         body_producer = _RequestBodyProducer(body.encode()) if body else None
         response: TxResponse = cast(
-            TxResponse,
+            "TxResponse",
             await maybe_deferred_to_future(
                 agent.request(
                     b"GET",
                     url.encode(),
-                    bodyProducer=cast(IBodyProducer, body_producer),
+                    bodyProducer=cast("IBodyProducer", body_producer),
                 )
             ),
         )

tests/test_crawl.py

@@ -5,7 +5,7 @@ import logging
 from ipaddress import IPv4Address
 from socket import gethostbyname
 from typing import TYPE_CHECKING, Any
-from urllib.parse import urlparse
+from urllib.parse import urlencode, urlparse
 
 import pytest
 from testfixtures import LogCapture
@@ -20,6 +20,7 @@ from scrapy.exceptions import CloseSpider, StopDownload
 from scrapy.http import Request
 from scrapy.http.response import Response
 from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.engine import format_engine_status, get_engine_status
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_crawler, get_reactor_settings
 from tests import NON_EXISTING_RESOLVABLE
@@ -255,8 +256,6 @@ class TestCrawl(TestCase):
     def test_unbounded_response(self):
         # Completeness of responses without Content-Length or Transfer-Encoding
         # can not be determined, we treat them as valid but flagged as "partial"
-        from urllib.parse import urlencode
-
         query = urlencode(
             {
                 "raw": """\
@@ -339,8 +338,6 @@ with multiples lines
 
     @inlineCallbacks
     def test_engine_status(self):
-        from scrapy.utils.engine import get_engine_status
-
         est = []
 
         def cb(response):
@@ -357,8 +354,6 @@ with multiples lines
 
     @inlineCallbacks
     def test_format_engine_status(self):
-        from scrapy.utils.engine import format_engine_status
-
         est = []
 
         def cb(response):

tests/test_crawler.py

@@ -526,10 +526,10 @@ class TestCrawlerLogging:
         crawler = get_crawler(MySpider)
         assert get_scrapy_root_handler().level == logging.INFO
         info_count = crawler.stats.get_value("log_count/INFO")
-        logging.debug("debug message")
+        logging.debug("debug message")  # noqa: LOG015
-        logging.info("info message")
+        logging.info("info message")  # noqa: LOG015
-        logging.warning("warning message")
+        logging.warning("warning message")  # noqa: LOG015
-        logging.error("error message")
+        logging.error("error message")  # noqa: LOG015
 
         logged = log_file.read_text(encoding="utf-8")
 
@@ -556,7 +556,7 @@ class TestCrawlerLogging:
 
         configure_logging()
         get_crawler(MySpider)
-        logging.debug("debug message")
+        logging.debug("debug message")  # noqa: LOG015
 
         logged = log_file.read_text(encoding="utf-8")
 

tests/test_downloader_handler_twisted_http2.py

@@ -15,10 +15,7 @@ from twisted.web.http import H2_ENABLED
 
 from scrapy.http import Request
 from scrapy.spiders import Spider
-from scrapy.utils.defer import (
+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
-    deferred_f_from_coro_f,
-    maybe_deferred_to_future,
-)
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
 from tests.mockserver import ssl_context_factory
@@ -46,7 +43,9 @@ class H2DownloadHandlerMixin:
     @property
     def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
         # the import can fail when H2_ENABLED is False
-        from scrapy.core.downloader.handlers.http2 import H2DownloadHandler
+        from scrapy.core.downloader.handlers.http2 import (  # noqa: PLC0415
+            H2DownloadHandler,
+        )
 
         return H2DownloadHandler
 

tests/test_downloader_handlers.py

@@ -13,7 +13,7 @@ from unittest import mock
 import pytest
 from twisted.cred import checkers, credentials, portal
 from twisted.internet.defer import inlineCallbacks
-from twisted.protocols.ftp import FTPFactory, FTPRealm
+from twisted.protocols.ftp import ConnectionLost, FTPFactory, FTPRealm
 from twisted.trial import unittest
 from w3lib.url import path_to_file_uri
 
@@ -27,10 +27,7 @@ from scrapy.http import HtmlResponse, Request, Response
 from scrapy.http.response.text import TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.spiders import Spider
-from scrapy.utils.defer import (
+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
-    deferred_f_from_coro_f,
-    maybe_deferred_to_future,
-)
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler
@@ -184,7 +181,7 @@ class TestS3:
     @contextlib.contextmanager
     def _mocked_date(self, date):
         try:
-            import botocore.auth  # noqa: F401
+            import botocore.auth  # noqa: F401,PLC0415
         except ImportError:
             yield
         else:
@@ -443,7 +440,6 @@ class TestFTP(TestFTPBase):
             pytest.skip(
                 "This test produces DirtyReactorAggregateError on Windows with asyncio"
             )
-        from twisted.protocols.ftp import ConnectionLost
 
         meta = dict(self.req_meta)
         meta.update({"ftp_password": "invalid"})

tests/test_downloadermiddleware_httpcompression.py

@@ -50,6 +50,23 @@ FORMAT = {
 }
 
 
+def _skip_if_no_br() -> None:
+    try:
+        try:
+            import brotli  # noqa: F401,PLC0415
+        except ImportError:
+            import brotlicffi  # noqa: F401,PLC0415
+    except ImportError:
+        pytest.skip("no brotli support")
+
+
+def _skip_if_no_zstd() -> None:
+    try:
+        import zstandard  # noqa: F401,PLC0415
+    except ImportError:
+        pytest.skip("no zstd support (zstandard)")
+
+
 class TestHttpCompression:
     def setup_method(self):
         self.crawler = get_crawler(Spider)
@@ -123,13 +140,8 @@ class TestHttpCompression:
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
     def test_process_response_br(self):
-        try:
+        _skip_if_no_br()
-            try:
+
-                import brotli  # noqa: F401
-            except ImportError:
-                import brotlicffi  # noqa: F401
-        except ImportError:
-            pytest.skip("no brotli")
         response = self._getresponse("br")
         request = response.request
         assert response.headers["Content-Encoding"] == b"br"
@@ -143,11 +155,11 @@ class TestHttpCompression:
     def test_process_response_br_unsupported(self):
         try:
             try:
-                import brotli  # noqa: F401
+                import brotli  # noqa: F401,PLC0415
 
                 pytest.skip("Requires not having brotli support")
             except ImportError:
-                import brotlicffi  # noqa: F401
+                import brotlicffi  # noqa: F401,PLC0415
 
                 pytest.skip("Requires not having brotli support")
         except ImportError:
@@ -176,10 +188,8 @@ class TestHttpCompression:
         assert newresponse.headers.getlist("Content-Encoding") == [b"br"]
 
     def test_process_response_zstd(self):
-        try:
+        _skip_if_no_zstd()
-            import zstandard  # noqa: F401
+
-        except ImportError:
-            pytest.skip("no zstd support (zstandard)")
         raw_content = None
         for check_key in FORMAT:
             if not check_key.startswith("zstd-"):
@@ -198,7 +208,7 @@ class TestHttpCompression:
 
     def test_process_response_zstd_unsupported(self):
         try:
-            import zstandard  # noqa: F401
+            import zstandard  # noqa: F401,PLC0415
 
             pytest.skip("Requires not having zstandard support")
         except ImportError:
@@ -513,13 +523,8 @@ class TestHttpCompression:
             mw.process_response(response.request, response, spider)
 
     def test_compression_bomb_setting_br(self):
-        try:
+        _skip_if_no_br()
-            try:
+
-                import brotli  # noqa: F401
-            except ImportError:
-                import brotlicffi  # noqa: F401
-        except ImportError:
-            pytest.skip("no brotli")
         self._test_compression_bomb_setting("br")
 
     def test_compression_bomb_setting_deflate(self):
@@ -529,10 +534,8 @@ class TestHttpCompression:
         self._test_compression_bomb_setting("gzip")
 
     def test_compression_bomb_setting_zstd(self):
-        try:
+        _skip_if_no_zstd()
-            import zstandard  # noqa: F401
+
-        except ImportError:
-            pytest.skip("no zstd support (zstandard)")
         self._test_compression_bomb_setting("zstd")
 
     def _test_compression_bomb_spider_attr(self, compression_id):
@@ -549,13 +552,8 @@ class TestHttpCompression:
             mw.process_response(response.request, response, spider)
 
     def test_compression_bomb_spider_attr_br(self):
-        try:
+        _skip_if_no_br()
-            try:
+
-                import brotli  # noqa: F401
-            except ImportError:
-                import brotlicffi  # noqa: F401
-        except ImportError:
-            pytest.skip("no brotli")
         self._test_compression_bomb_spider_attr("br")
 
     def test_compression_bomb_spider_attr_deflate(self):
@@ -565,10 +563,8 @@ class TestHttpCompression:
         self._test_compression_bomb_spider_attr("gzip")
 
     def test_compression_bomb_spider_attr_zstd(self):
-        try:
+        _skip_if_no_zstd()
-            import zstandard  # noqa: F401
+
-        except ImportError:
-            pytest.skip("no zstd support (zstandard)")
         self._test_compression_bomb_spider_attr("zstd")
 
     def _test_compression_bomb_request_meta(self, compression_id):
@@ -583,13 +579,8 @@ class TestHttpCompression:
             mw.process_response(response.request, response, spider)
 
     def test_compression_bomb_request_meta_br(self):
-        try:
+        _skip_if_no_br()
-            try:
+
-                import brotli  # noqa: F401
-            except ImportError:
-                import brotlicffi  # noqa: F401
-        except ImportError:
-            pytest.skip("no brotli")
         self._test_compression_bomb_request_meta("br")
 
     def test_compression_bomb_request_meta_deflate(self):
@@ -599,10 +590,8 @@ class TestHttpCompression:
         self._test_compression_bomb_request_meta("gzip")
 
     def test_compression_bomb_request_meta_zstd(self):
-        try:
+        _skip_if_no_zstd()
-            import zstandard  # noqa: F401
+
-        except ImportError:
-            pytest.skip("no zstd support (zstandard)")
         self._test_compression_bomb_request_meta("zstd")
 
     def _test_download_warnsize_setting(self, compression_id):
@@ -632,13 +621,8 @@ class TestHttpCompression:
         )
 
     def test_download_warnsize_setting_br(self):
-        try:
+        _skip_if_no_br()
-            try:
+
-                import brotli  # noqa: F401
-            except ImportError:
-                import brotlicffi  # noqa: F401
-        except ImportError:
-            pytest.skip("no brotli")
         self._test_download_warnsize_setting("br")
 
     def test_download_warnsize_setting_deflate(self):
@@ -648,10 +632,8 @@ class TestHttpCompression:
         self._test_download_warnsize_setting("gzip")
 
     def test_download_warnsize_setting_zstd(self):
-        try:
+        _skip_if_no_zstd()
-            import zstandard  # noqa: F401
+
-        except ImportError:
-            pytest.skip("no zstd support (zstandard)")
         self._test_download_warnsize_setting("zstd")
 
     def _test_download_warnsize_spider_attr(self, compression_id):
@@ -683,13 +665,8 @@ class TestHttpCompression:
         )
 
     def test_download_warnsize_spider_attr_br(self):
-        try:
+        _skip_if_no_br()
-            try:
+
-                import brotli  # noqa: F401
-            except ImportError:
-                import brotlicffi  # noqa: F401
-        except ImportError:
-            pytest.skip("no brotli")
         self._test_download_warnsize_spider_attr("br")
 
     def test_download_warnsize_spider_attr_deflate(self):
@@ -699,10 +676,8 @@ class TestHttpCompression:
         self._test_download_warnsize_spider_attr("gzip")
 
     def test_download_warnsize_spider_attr_zstd(self):
-        try:
+        _skip_if_no_zstd()
-            import zstandard  # noqa: F401
+
-        except ImportError:
-            pytest.skip("no zstd support (zstandard)")
         self._test_download_warnsize_spider_attr("zstd")
 
     def _test_download_warnsize_request_meta(self, compression_id):
@@ -732,13 +707,8 @@ class TestHttpCompression:
         )
 
     def test_download_warnsize_request_meta_br(self):
-        try:
+        _skip_if_no_br()
-            try:
+
-                import brotli  # noqa: F401
-            except ImportError:
-                import brotlicffi  # noqa: F401
-        except ImportError:
-            pytest.skip("no brotli")
         self._test_download_warnsize_request_meta("br")
 
     def test_download_warnsize_request_meta_deflate(self):
@@ -748,8 +718,6 @@ class TestHttpCompression:
         self._test_download_warnsize_request_meta("gzip")
 
     def test_download_warnsize_request_meta_zstd(self):
-        try:
+        _skip_if_no_zstd()
-            import zstandard  # noqa: F401
+
-        except ImportError:
-            pytest.skip("no zstd support (zstandard)")
         self._test_download_warnsize_request_meta("zstd")

tests/test_feedexport.py

@@ -5,7 +5,9 @@ import csv
 import gzip
 import json
 import lzma
+import marshal
 import os
+import pickle
 import random
 import shutil
 import string
@@ -25,6 +27,7 @@ from urllib.request import pathname2url
 
 import lxml.etree
 import pytest
+from packaging.version import Version
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.internet.defer import inlineCallbacks
@@ -79,7 +82,7 @@ def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
     """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
     classes and set their proper return values.
     """
-    from google.cloud.storage import Blob, Bucket, Client
+    from google.cloud.storage import Blob, Bucket, Client  # noqa: PLC0415
 
     client_mock = mock.create_autospec(Client)
 
@@ -507,7 +510,7 @@ class TestS3FeedStorage(unittest.TestCase):
 class TestGCSFeedStorage(unittest.TestCase):
     def test_parse_settings(self):
         try:
-            from google.cloud.storage import Client  # noqa: F401
+            from google.cloud.storage import Client  # noqa: F401,PLC0415
         except ImportError:
             pytest.skip("GCSFeedStorage requires google-cloud-storage")
 
@@ -521,7 +524,7 @@ class TestGCSFeedStorage(unittest.TestCase):
 
     def test_parse_empty_acl(self):
         try:
-            from google.cloud.storage import Client  # noqa: F401
+            from google.cloud.storage import Client  # noqa: F401,PLC0415
         except ImportError:
             pytest.skip("GCSFeedStorage requires google-cloud-storage")
 
@@ -538,7 +541,7 @@ class TestGCSFeedStorage(unittest.TestCase):
     @deferred_f_from_coro_f
     async def test_store(self):
         try:
-            from google.cloud.storage import Client  # noqa: F401
+            from google.cloud.storage import Client  # noqa: F401,PLC0415
         except ImportError:
             pytest.skip("GCSFeedStorage requires google-cloud-storage")
 
@@ -976,7 +979,6 @@ class TestFeedExport(TestFeedExportBase):
         )
         data = await self.exported_data(items, settings)
         expected = [{k: v for k, v in row.items() if v} for row in rows]
-        import pickle
 
         result = self._load_until_eof(data["pickle"], load_func=pickle.load)
         assert result == expected
@@ -997,7 +999,6 @@ class TestFeedExport(TestFeedExportBase):
         )
         data = await self.exported_data(items, settings)
         expected = [{k: v for k, v in row.items() if v} for row in rows]
-        import marshal
 
         result = self._load_until_eof(data["marshal"], load_func=marshal.load)
         assert result == expected
@@ -2300,9 +2301,6 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
 
     @deferred_f_from_coro_f
     async def test_exports_compatibility_with_postproc(self):
-        import marshal
-        import pickle
-
         filename_to_expected = {
             self._named_tempfile("csv"): b"foo\r\nbar\r\n",
             self._named_tempfile("json"): b'[\n{"foo": "bar"}\n]',
@@ -2484,7 +2482,6 @@ class TestBatchDeliveries(TestFeedExportBase):
         batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = await self.exported_data(items, settings)
-        import pickle
 
         for batch in data["pickle"]:
             got_batch = self._load_until_eof(batch, load_func=pickle.load)
@@ -2505,7 +2502,6 @@ class TestBatchDeliveries(TestFeedExportBase):
         batch_size = Settings(settings).getint("FEED_EXPORT_BATCH_ITEM_COUNT")
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = await self.exported_data(items, settings)
-        import marshal
 
         for batch in data["marshal"]:
             got_batch = self._load_until_eof(batch, load_func=marshal.load)
@@ -2714,9 +2710,8 @@ class TestBatchDeliveries(TestFeedExportBase):
             stubs = []
 
             def open(self, *args, **kwargs):
-                from botocore import __version__ as botocore_version
+                from botocore import __version__ as botocore_version  # noqa: PLC0415
-                from botocore.stub import ANY, Stubber
+                from botocore.stub import ANY, Stubber  # noqa: PLC0415
-                from packaging.version import Version
 
                 expected_params = {
                     "Body": ANY,
@@ -2912,10 +2907,9 @@ class TestURIParams(ABC):
                 match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated",
             ):
                 crawler = get_crawler(settings_dict=settings)
-                feed_exporter = FeedExporter.from_crawler(crawler)
         else:
             crawler = get_crawler(settings_dict=settings)
-            feed_exporter = FeedExporter.from_crawler(crawler)
+        feed_exporter = crawler.get_extension(FeedExporter)
         return crawler, feed_exporter
 
     def test_default(self):

tests/test_http2_client_protocol.py

@@ -243,7 +243,7 @@ class TestHttps2ClientProtocol(TestCase):
 
         self.conn_closed_deferred = Deferred()
 
-        from scrapy.core.http2.protocol import H2ClientFactory
+        from scrapy.core.http2.protocol import H2ClientFactory  # noqa: PLC0415
 
         h2_client_factory = H2ClientFactory(uri, Settings(), self.conn_closed_deferred)
         client_endpoint = SSL4ClientEndpoint(
@@ -445,7 +445,7 @@ class TestHttps2ClientProtocol(TestCase):
     def test_received_dataloss_response(self):
         """In case when value of Header Content-Length != len(Received Data)
         ProtocolError is raised"""
-        from h2.exceptions import InvalidBodyLengthError
+        from h2.exceptions import InvalidBodyLengthError  # noqa: PLC0415
 
         request = Request(url=self.get_url("/dataloss"))
         with pytest.raises(ResponseFailed) as exc_info:
@@ -525,7 +525,7 @@ class TestHttps2ClientProtocol(TestCase):
         def assert_inactive_stream(failure):
             assert failure.check(ResponseFailed) is not None
 
-            from scrapy.core.http2.stream import InactiveStreamClosed
+            from scrapy.core.http2.stream import InactiveStreamClosed  # noqa: PLC0415
 
             assert any(
                 isinstance(e, InactiveStreamClosed) for e in failure.value.reasons
@@ -593,7 +593,7 @@ class TestHttps2ClientProtocol(TestCase):
         assert str(response.ip_address) == "127.0.0.1"
 
     async def _check_invalid_netloc(self, url: str) -> None:
-        from scrapy.core.http2.stream import InvalidHostname
+        from scrapy.core.http2.stream import InvalidHostname  # noqa: PLC0415
 
         request = Request(url)
         with pytest.raises(InvalidHostname) as exc_info:
@@ -630,7 +630,7 @@ class TestHttps2ClientProtocol(TestCase):
             yield self.make_request_dfd(request)
 
         for err in exc_info.value.reasons:
-            from scrapy.core.http2.protocol import H2ClientProtocol
+            from scrapy.core.http2.protocol import H2ClientProtocol  # noqa: PLC0415
 
             if isinstance(err, TimeoutError):
                 assert (

tests/test_http_request.py

@@ -1441,7 +1441,7 @@ class TestXmlRpcRequest(TestRequest):
         )
         assert r.method == "POST"
         assert r.encoding == kwargs.get("encoding", "utf-8")
-        assert r.dont_filter, True
+        assert r.dont_filter
 
     def test_xmlrpc_dumps(self):
         self._test_request(params=("value",))

tests/test_logformatter.py

@@ -110,7 +110,7 @@ class TestLogFormatter:
         assert logkws["level"] == unsupported_value
 
         with pytest.raises(TypeError):
-            logging.log(logkws["level"], "message")
+            logging.log(logkws["level"], "message")  # noqa: LOG015
 
     def test_dropitem_custom_log_level(self):
         item = {}

tests/test_pipeline_files.py

@@ -5,6 +5,7 @@ import time
 import warnings
 from abc import ABC, abstractmethod
 from datetime import datetime
+from ftplib import FTP
 from io import BytesIO
 from pathlib import Path
 from posixpath import split
@@ -29,9 +30,7 @@ from scrapy.pipelines.files import (
     GCSFilesStore,
     S3FilesStore,
 )
-from scrapy.utils.test import (
+from scrapy.utils.test import get_crawler
-    get_crawler,
-)
 from tests.mockserver import MockFTPServer
 
 from .test_pipeline_media import _mocked_download_func
@@ -40,7 +39,7 @@ from .test_pipeline_media import _mocked_download_func
 def get_gcs_content_and_delete(
     bucket: Any, path: str
 ) -> tuple[bytes, list[dict[str, str]], Any]:
-    from google.cloud import storage
+    from google.cloud import storage  # noqa: PLC0415
 
     client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))
     bucket = client.get_bucket(bucket)
@@ -59,8 +58,6 @@ def get_ftp_content_and_delete(
     password: str,
     use_active_mode: bool = False,
 ) -> bytes:
-    from ftplib import FTP
-
     ftp = FTP()
     ftp.connect(host, port)
     ftp.login(username, password)
@@ -553,7 +550,7 @@ class TestS3FilesStore(unittest.TestCase):
         content_type = "image/png"
 
         store = S3FilesStore(uri)
-        from botocore.stub import Stubber
+        from botocore.stub import Stubber  # noqa: PLC0415
 
         with Stubber(store.s3_client) as stub:
             stub.add_response(
@@ -591,7 +588,7 @@ class TestS3FilesStore(unittest.TestCase):
         last_modified = datetime(2019, 12, 1)
 
         store = S3FilesStore(uri)
-        from botocore.stub import Stubber
+        from botocore.stub import Stubber  # noqa: PLC0415
 
         with Stubber(store.s3_client) as stub:
             stub.add_response(
@@ -650,7 +647,7 @@ class TestGCSFilesStore(unittest.TestCase):
         already uploaded files.
         """
         try:
-            import google.cloud.storage  # noqa: F401
+            import google.cloud.storage  # noqa: F401,PLC0415
         except ModuleNotFoundError:
             pytest.skip("google-cloud-storage is not installed")
         with (

tests/test_proxy_connect.py

@@ -74,7 +74,7 @@ class TestProxyConnect(TestCase):
 
     def setUp(self):
         try:
-            import mitmproxy  # noqa: F401
+            import mitmproxy  # noqa: F401,PLC0415
         except ImportError:
             pytest.skip("mitmproxy is not installed")
 

tests/test_robotstxt_interface.py

@@ -1,21 +1,19 @@
 import pytest
 
-from scrapy.robotstxt import decode_robotstxt
+from scrapy.robotstxt import (
+    ProtegoRobotParser,
+    PythonRobotParser,
+    RerpRobotParser,
+    decode_robotstxt,
+)
 
 
 def rerp_available():
     # check if robotexclusionrulesparser is installed
     try:
-        from robotexclusionrulesparser import RobotExclusionRulesParser  # noqa: F401
+        from robotexclusionrulesparser import (  # noqa: PLC0415
-    except ImportError:
+            RobotExclusionRulesParser,  # noqa: F401
-        return False
+        )
-    return True
-
-
-def protego_available():
-    # check if protego parser is installed
-    try:
-        from protego import Protego  # noqa: F401
     except ImportError:
         return False
     return True
@@ -134,8 +132,6 @@ class TestDecodeRobotsTxt:
 
 class TestPythonRobotParser(BaseRobotParserTest):
     def setup_method(self):
-        from scrapy.robotstxt import PythonRobotParser
-
         super()._setUp(PythonRobotParser)
 
     def test_length_based_precedence(self):
@@ -150,19 +146,14 @@ class TestPythonRobotParser(BaseRobotParserTest):
 @pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")
 class TestRerpRobotParser(BaseRobotParserTest):
     def setup_method(self):
-        from scrapy.robotstxt import RerpRobotParser
-
         super()._setUp(RerpRobotParser)
 
     def test_length_based_precedence(self):
         pytest.skip("Rerp does not support length based directives precedence.")
 
 
-@pytest.mark.skipif(not protego_available(), reason="Protego parser is not installed")
 class TestProtegoRobotParser(BaseRobotParserTest):
     def setup_method(self):
-        from scrapy.robotstxt import ProtegoRobotParser
-
         super()._setUp(ProtegoRobotParser)
 
     def test_order_based_precedence(self):

tests/test_scrapy__getattr__.py

@@ -3,7 +3,9 @@ import warnings
 
 def test_deprecated_twisted_version():
     with warnings.catch_warnings(record=True) as warns:
-        from scrapy import twisted_version  # pylint: disable=no-name-in-module
+        from scrapy import (  # noqa: PLC0415  # pylint: disable=no-name-in-module
+            twisted_version,
+        )
 
         assert twisted_version is not None
         assert isinstance(twisted_version, tuple)
@@ -15,7 +17,7 @@ def test_deprecated_twisted_version():
 
 def test_deprecated_concurrent_requests_per_ip_attribute():
     with warnings.catch_warnings(record=True) as warns:
-        from scrapy.settings.default_settings import (
+        from scrapy.settings.default_settings import (  # noqa: PLC0415
             CONCURRENT_REQUESTS_PER_IP,
         )
 

tests/test_settings/__init__.py

@@ -6,6 +6,7 @@ from unittest import mock
 
 import pytest
 
+from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.settings import (
     SETTINGS_PRIORITIES,
     BaseSettings,
@@ -13,6 +14,8 @@ from scrapy.settings import (
     SettingsAttribute,
     get_settings_priority,
 )
+from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.test import get_crawler
 
 from . import default_settings
 
@@ -447,10 +450,6 @@ class TestSettings:
         assert mydict["key"] == "val"
 
     def test_passing_objects_as_values(self):
-        from scrapy.core.downloader.handlers.file import FileDownloadHandler
-        from scrapy.utils.misc import build_from_crawler
-        from scrapy.utils.test import get_crawler
-
         class TestPipeline:
             def process_item(self, i, s):
                 return i

tests/test_spider.py

@@ -1,7 +1,9 @@
 from __future__ import annotations
 
 import gzip
+import re
 import warnings
+from datetime import datetime
 from io import BytesIO
 from logging import ERROR, WARNING
 from pathlib import Path
@@ -297,8 +299,6 @@ class TestCrawlSpider(TestSpider):
         )
 
         class _CrawlSpider(self.spider_class):
-            import re
-
             name = "test"
             allowed_domains = ["example.org"]
             rules = (Rule(LinkExtractor(), process_links="filter_process_links"),)
@@ -635,8 +635,6 @@ Sitemap: /sitemap-relative-url.xml
 
         class FilteredSitemapSpider(self.spider_class):
             def sitemap_filter(self, entries):
-                from datetime import datetime
-
                 for entry in entries:
                     date_time = datetime.strptime(entry["lastmod"], "%Y-%m-%d")
                     if date_time.year > 2008:
@@ -706,8 +704,6 @@ Sitemap: /sitemap-relative-url.xml
 
         class FilteredSitemapSpider(self.spider_class):
             def sitemap_filter(self, entries):
-                from datetime import datetime
-
                 for entry in entries:
                     date_time = datetime.strptime(
                         entry["lastmod"].split("T")[0], "%Y-%m-%d"

tests/test_spidermiddleware_referer.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import warnings
-from typing import Any, cast
+from typing import TYPE_CHECKING, Any, cast
 from urllib.parse import urlparse
 
 import pytest
@@ -33,6 +33,9 @@ from scrapy.spidermiddlewares.referer import (
 )
 from scrapy.spiders import Spider
 
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
 
 @pytest.fixture
 def spider() -> Spider:
@@ -971,7 +974,7 @@ class TestPolicyHeaderPrecedence004(
 
 class TestReferrerOnRedirect(TestRefererMiddleware):
     settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.UnsafeUrlPolicy"}
-    scenarii: list[
+    scenarii: Sequence[
         tuple[str, str, tuple[tuple[int, str], ...], bytes | None, bytes | None]
     ] = [  # type: ignore[assignment]
         (
@@ -1041,7 +1044,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
                     request.url, headers={"Location": url}, status=status
                 )
                 request = cast(
-                    Request, redirectmw.process_response(request, response, spider)
+                    "Request", redirectmw.process_response(request, response, spider)
                 )
                 referrermw.request_scheduled(request, spider)
 

tests/test_toplevel.py

@@ -10,26 +10,26 @@ def test_version_info():
 
 
 def test_request_shortcut():
-    from scrapy.http import FormRequest, Request
+    from scrapy.http import FormRequest, Request  # noqa: PLC0415
 
     assert scrapy.Request is Request
     assert scrapy.FormRequest is FormRequest
 
 
 def test_spider_shortcut():
-    from scrapy.spiders import Spider
+    from scrapy.spiders import Spider  # noqa: PLC0415
 
     assert scrapy.Spider is Spider
 
 
 def test_selector_shortcut():
-    from scrapy.selector import Selector
+    from scrapy.selector import Selector  # noqa: PLC0415
 
     assert scrapy.Selector is Selector
 
 
 def test_item_shortcut():
-    from scrapy.item import Field, Item
+    from scrapy.item import Field, Item  # noqa: PLC0415
 
     assert scrapy.Item is Item
     assert scrapy.Field is Field

tests/test_utils_display.py

@@ -1,3 +1,4 @@
+import builtins
 from io import StringIO
 from unittest import mock
 
@@ -74,8 +75,6 @@ def test_pformat_windows(isatty, version, terminal_processing):
 def test_pformat_no_pygments(isatty):
     isatty.return_value = True
 
-    import builtins
-
     real_import = builtins.__import__
 
     def mock_import(name, globals, locals, fromlist, level):

tests/test_utils_spider.py

@@ -24,7 +24,7 @@ def test_iterate_spider_output():
 
 
 def test_iter_spider_classes():
-    import tests.test_utils_spider  # noqa: PLW0406  # pylint: disable=import-self
+    import tests.test_utils_spider  # noqa: PLW0406,PLC0415  # pylint: disable=import-self
 
     it = iter_spider_classes(tests.test_utils_spider)
     assert set(it) == {MySpider1, MySpider2}

tests/test_utils_url.py

@@ -1,4 +1,5 @@
 import warnings
+from importlib import import_module
 
 import pytest
 
@@ -461,8 +462,6 @@ def test_deprecated_imports_from_w3lib(obj_name: str) -> None:
         obj_type = "attribute" if obj_name == "_safe_chars" else "function"
         message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."
 
-        from importlib import import_module
-
         getattr(import_module("scrapy.utils.url"), obj_name)
 
         assert isinstance(warns[0].message, Warning)

tests/test_webclient.py

@@ -18,11 +18,10 @@ from twisted.internet.testing import StringTransport
 from twisted.protocols.policies import WrappingFactory
 from twisted.trial import unittest
 from twisted.web import resource, server, static, util
+from twisted.web.client import _makeGetterFactory
 
 from scrapy.core.downloader import webclient as client
-from scrapy.core.downloader.contextfactory import (
+from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
-    ScrapyClientContextFactory,
-)
 from scrapy.http import Headers, Request
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.python import to_bytes, to_unicode
@@ -51,8 +50,6 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
         f.deferred.addCallback(response_transform or (lambda r: r.body))
         return f
 
-    from twisted.web.client import _makeGetterFactory
-
     return _makeGetterFactory(
         to_bytes(url),
         _clientfactory,

tox.ini

@@ -46,17 +46,21 @@ install_command =
 [testenv:typing]
 basepython = python3.9
 deps =
-    mypy==1.14.0
+    mypy==1.16.1
-    typing-extensions==4.12.2
+    typing-extensions==4.14.1
-    types-lxml==2024.12.13
+    types-defusedxml==0.7.0.20250516
-    types-Pygments==2.18.0.20240506
+    types-lxml==2025.3.30
-    botocore-stubs==1.35.90
+    types-pexpect==4.9.0.20250516
-    boto3-stubs[s3]==1.35.90
+    types-Pygments==2.19.0.20250516
+    botocore-stubs==1.38.46
+    boto3-stubs[s3]==1.39.3
+    itemadapter==0.11.0
+    Protego==0.5.0
+    w3lib==2.3.1
     attrs >= 18.2.0
     Pillow >= 10.3.0
     pyOpenSSL >= 24.2.1
     pytest >= 8.2.0
-    w3lib >= 2.2.0
 commands =
     mypy {posargs:scrapy tests}
 
@@ -80,14 +84,14 @@ commands =
 basepython = python3
 deps =
     {[testenv:extra-deps]deps}
-    pylint==3.3.3
+    pylint==3.3.7
 commands =
     pylint conftest.py docs extras scrapy tests
 
 [testenv:twinecheck]
 basepython = python3
 deps =
-    twine==6.0.1
+    twine==6.1.0
     build==1.2.2.post1
 commands =
     python -m build --sdist