chore: more ruff rules and overall minor improvements (#7386)

* overall prettyfication

- don't create empty mutable containers (lists, dicts) where it is appropriate
- removed from ignore section and applied some rules from ruff (but keep them ignored in tests)
- use `deque` in `_AsyncCooperatorAdapter` instead of `list.pop(0)`
- remove `f` prefix from strings without any formatting

* return `SIM300` to ignore

* apply ruff rules to all files

* another one

* remove extra space in pyproject.toml

* simplify `__getattr__` in `scrapy.utils.url`

* lazy `url_is_from_any_domain` and `url_is_from_spider`

* improve typing

* specify `arg_to_iter` with None as arg

pyproject.toml

@@ -415,25 +415,6 @@ ignore = [
     "SIM115",
     # Yoda condition detected
     "SIM300",
-
-    # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
-
-    # Assigning to `os.environ` doesn't clear the environment.
-    "B003",
-    # Do not use mutable data structures for argument defaults.
-    "B006",
-    # Found useless expression.
-    "B018",
-    # No explicit stacklevel argument found.
-    "B028",
-    # Within an `except` clause, raise exceptions with `raise ... from`
-    "B904",
-    # `for` loop variable overwritten by assignment target
-    "PLW2901",
-    # Mutable class attributes should be annotated with `typing.ClassVar`
-    "RUF012",
-    # Use capitalized environment variable
-    "SIM112",
 ]
 
 [tool.ruff.lint.flake8-tidy-imports]
@@ -453,8 +434,28 @@ split-on-trailing-comma = false
 "scrapy/linkextractors/__init__.py" = ["E402"]
 "scrapy/spiders/__init__.py" = ["E402"]
 
-# Skip bandit and allow blocking file I/O in tests
+"tests/**" = [
-"tests/**" = ["ASYNC240", "S"]
+    # Skip bandit and allow blocking file I/O in tests
+    "ASYNC240",
+    "S",
+    # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
+    # Assigning to `os.environ` doesn't clear the environment.
+    "B003",
+    # Do not use mutable data structures for argument defaults.
+    "B006",
+    # Found useless expression.
+    "B018",
+    # No explicit stacklevel argument found.
+    "B028",
+    # Within an `except` clause, raise exceptions with `raise ... from`
+    "B904",
+    # `for` loop variable overwritten by assignment target
+    "PLW2901",
+    # Mutable class attributes should be annotated with `typing.ClassVar`
+    "RUF012",
+    # Use capitalized environment variable
+    "SIM112",
+]
 
 # Issues pending a review:
 "docs/conf.py" = ["E402"]

scrapy/cmdline.py

@@ -46,7 +46,7 @@ def _iter_command_classes(module_name: str) -> Iterable[type[ScrapyCommand]]:
                 inspect.isclass(obj)
                 and issubclass(obj, ScrapyCommand)
                 and obj.__module__ == module.__name__
-                and obj not in (ScrapyCommand, BaseRunSpiderCommand)
+                and obj not in {ScrapyCommand, BaseRunSpiderCommand}
             ):
                 yield obj
 

scrapy/commands/__init__.py

@@ -10,7 +10,7 @@ import os
 import warnings
 from abc import ABC, abstractmethod
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, ClassVar
 
 from twisted.python import failure
 
@@ -30,7 +30,7 @@ class ScrapyCommand(ABC):
     crawler_process: CrawlerProcessBase | None = None  # set in scrapy.cmdline
 
     # default settings to be used for this command instead of global defaults
-    default_settings: dict[str, Any] = {}
+    default_settings: ClassVar[dict[str, Any]] = {}
 
     exitcode: int = 0
 
@@ -115,7 +115,9 @@ class ScrapyCommand(ABC):
         try:
             self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline")
         except ValueError:
-            raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
+            raise UsageError(
+                "Invalid -s value, use -s NAME=VALUE", print_help=False
+            ) from None
 
         if opts.logfile:
             self.settings.set("LOG_ENABLED", True, priority="cmdline")
@@ -181,7 +183,9 @@ class BaseRunSpiderCommand(ScrapyCommand):
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
-            raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
+            raise UsageError(
+                "Invalid -a value, use -a NAME=VALUE", print_help=False
+            ) from None
         if opts.output or opts.overwrite_output:
             assert self.settings is not None
             feeds = feed_process_params_from_cli(

scrapy/commands/bench.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import subprocess
 import sys
 import time
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, ClassVar
 from urllib.parse import urlencode
 
 import scrapy
@@ -18,7 +18,7 @@ if TYPE_CHECKING:
 
 
 class Command(ScrapyCommand):
-    default_settings = {
+    default_settings: ClassVar[dict[str, Any]] = {
         "LOG_LEVEL": "INFO",
         "LOGSTATS_INTERVAL": 1,
         "CLOSESPIDER_TIMEOUT": 10,

scrapy/commands/check.py

@@ -2,7 +2,7 @@ import argparse
 import time
 from collections import defaultdict
 from collections.abc import AsyncIterator
-from typing import Any
+from typing import Any, ClassVar
 from unittest import TextTestResult as _TextTestResult
 from unittest import TextTestRunner
 
@@ -44,7 +44,7 @@ class TextTestResult(_TextTestResult):
 
 class Command(ScrapyCommand):
     requires_project = True
-    default_settings = {"LOG_ENABLED": False}
+    default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
 
     def syntax(self) -> str:
         return "[options] <spider>"

scrapy/commands/edit.py

@@ -1,6 +1,7 @@
 import argparse
 import os
 import sys
+from typing import Any, ClassVar
 
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
@@ -10,7 +11,7 @@ from scrapy.spiderloader import get_spider_loader
 class Command(ScrapyCommand):
     requires_project = True
     requires_crawler_process = False
-    default_settings = {"LOG_ENABLED": False}
+    default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
 
     def syntax(self) -> str:
         return "<spider>"

scrapy/commands/genspider.py

@@ -5,7 +5,7 @@ import shutil
 import string
 from importlib import import_module
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any, ClassVar, cast
 from urllib.parse import urlparse
 
 import scrapy
@@ -47,7 +47,7 @@ def verify_url_scheme(url: str) -> str:
 
 class Command(ScrapyCommand):
     requires_crawler_process = False
-    default_settings = {"LOG_ENABLED": False}
+    default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
 
     def syntax(self) -> str:
         return "[options] <name> <domain>"

scrapy/commands/list.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, ClassVar
 
 from scrapy.commands import ScrapyCommand
 from scrapy.spiderloader import get_spider_loader
@@ -12,7 +12,7 @@ if TYPE_CHECKING:
 class Command(ScrapyCommand):
     requires_project = True
     requires_crawler_process = False
-    default_settings = {"LOG_ENABLED": False}
+    default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
 
     def short_desc(self) -> str:
         return "List available spiders"

scrapy/commands/parse.py

@@ -4,7 +4,7 @@ import functools
 import inspect
 import json
 import logging
-from typing import TYPE_CHECKING, Any, TypeVar, overload
+from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, overload
 
 from itemadapter import ItemAdapter
 from twisted.internet.defer import Deferred, maybeDeferred
@@ -39,8 +39,8 @@ class Command(BaseRunSpiderCommand):
     requires_project = True
 
     spider: Spider | None = None
-    items: dict[int, list[Any]] = {}
+    items: ClassVar[dict[int, list[Any]]] = {}
-    requests: dict[int, list[Request]] = {}
+    requests: ClassVar[dict[int, list[Request]]] = {}
     spidercls: type[Spider] | None
 
     first_response = None
@@ -387,7 +387,7 @@ class Command(BaseRunSpiderCommand):
                     "Invalid -m/--meta value, pass a valid json string to -m or --meta. "
                     'Example: --meta=\'{"foo" : "bar"}\'',
                     print_help=False,
-                )
+                ) from None
 
     def process_request_cb_kwargs(self, opts: argparse.Namespace) -> None:
         if opts.cbkwargs:
@@ -398,7 +398,7 @@ class Command(BaseRunSpiderCommand):
                     "Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
                     'Example: --cbkwargs=\'{"foo" : "bar"}\'',
                     print_help=False,
-                )
+                ) from None
 
     def run(self, args: list[str], opts: argparse.Namespace) -> None:
         # parse arguments

scrapy/commands/runspider.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import sys
 from importlib import import_module
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, ClassVar
 
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
@@ -18,7 +18,7 @@ if TYPE_CHECKING:
 
 def _import_file(filepath: str | PathLike[str]) -> ModuleType:
     abspath = Path(filepath).resolve()
-    if abspath.suffix not in (".py", ".pyw"):
+    if abspath.suffix not in {".py", ".pyw"}:
         raise ValueError(f"Not a Python source file: {abspath}")
     dirname = str(abspath.parent)
     sys.path = [dirname, *sys.path]
@@ -30,7 +30,9 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType:
 
 
 class Command(BaseRunSpiderCommand):
-    default_settings = {"SPIDER_LOADER_CLASS": DummySpiderLoader}
+    default_settings: ClassVar[dict[str, Any]] = {
+        "SPIDER_LOADER_CLASS": DummySpiderLoader
+    }
 
     def syntax(self) -> str:
         return "[options] <spider_file>"
@@ -50,7 +52,7 @@ class Command(BaseRunSpiderCommand):
         try:
             module = _import_file(filename)
         except (ImportError, ValueError) as e:
-            raise UsageError(f"Unable to load {str(filename)!r}: {e}\n")
+            raise UsageError(f"Unable to load {str(filename)!r}: {e}\n") from e
         spclasses = list(iter_spider_classes(module))
         if not spclasses:
             raise UsageError(f"No spider found in file: {filename}\n")

scrapy/commands/settings.py

@@ -1,5 +1,6 @@
 import argparse
 import json
+from typing import Any, ClassVar
 
 from scrapy.commands import ScrapyCommand
 from scrapy.settings import BaseSettings
@@ -7,7 +8,7 @@ from scrapy.settings import BaseSettings
 
 class Command(ScrapyCommand):
     requires_crawler_process = False
-    default_settings = {"LOG_ENABLED": False}
+    default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
 
     def syntax(self) -> str:
         return "[options]"

scrapy/commands/shell.py

@@ -7,7 +7,7 @@ See documentation in docs/topics/shell.rst
 from __future__ import annotations
 
 from threading import Thread
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, ClassVar
 
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
@@ -23,7 +23,7 @@ if TYPE_CHECKING:
 
 
 class Command(ScrapyCommand):
-    default_settings = {
+    default_settings: ClassVar[dict[str, Any]] = {
         "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
         "KEEP_ALIVE": True,
         "LOGSTATS_INTERVAL": 0,

scrapy/commands/startproject.py

@@ -6,7 +6,7 @@ from importlib.util import find_spec
 from pathlib import Path
 from shutil import copy2, copystat, ignore_patterns, move
 from stat import S_IWUSR as OWNER_WRITE_PERMISSION
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, ClassVar
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -34,7 +34,7 @@ def _make_writable(path: Path) -> None:
 
 class Command(ScrapyCommand):
     requires_crawler_process = False
-    default_settings = {"LOG_ENABLED": False}
+    default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
 
     def syntax(self) -> str:
         return "<project_name> [project_dir]"
@@ -90,7 +90,7 @@ class Command(ScrapyCommand):
         _make_writable(dst)
 
     def run(self, args: list[str], opts: argparse.Namespace) -> None:
-        if len(args) not in (1, 2):
+        if len(args) not in {1, 2}:
             raise UsageError
 
         project_name = args[0]

scrapy/commands/version.py

@@ -1,4 +1,5 @@
 import argparse
+from typing import Any, ClassVar
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -7,7 +8,7 @@ from scrapy.utils.versions import get_versions
 
 class Command(ScrapyCommand):
     requires_crawler_process = False
-    default_settings = {"LOG_ENABLED": False}
+    default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
 
     def syntax(self) -> str:
         return "[-v]"

scrapy/contracts/__init__.py

@@ -6,7 +6,7 @@ from collections.abc import AsyncGenerator, Iterable
 from functools import wraps
 from inspect import getmembers
 from types import CoroutineType
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any, ClassVar, cast
 from unittest import TestCase, TestResult
 
 from scrapy.http import Request, Response
@@ -90,7 +90,7 @@ class Contract:
 
 
 class ContractsManager:
-    contracts: dict[str, type[Contract]] = {}
+    contracts: ClassVar[dict[str, type[Contract]]] = {}
 
     def __init__(self, contracts: Iterable[type[Contract]]):
         for contract in contracts:
@@ -108,8 +108,8 @@ class ContractsManager:
     def extract_contracts(self, method: Callable) -> list[Contract]:
         contracts: list[Contract] = []
         assert method.__doc__ is not None
-        for line in method.__doc__.split("\n"):
+        for line_ in method.__doc__.split("\n"):
-            line = line.strip()
+            line = line_.strip()
 
             if line.startswith("@"):
                 m = re.match(r"@(\w+)\s*(.*)", line)

scrapy/contracts/default.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import json
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, ClassVar
 
 from itemadapter import ItemAdapter, is_item
 
@@ -68,7 +68,7 @@ class ReturnsContract(Contract):
     """
 
     name = "returns"
-    object_type_verifiers: dict[str | None, Callable[[Any], bool]] = {
+    object_type_verifiers: ClassVar[dict[str | None, Callable[[Any], bool]]] = {
         "request": lambda x: isinstance(x, Request),
         "requests": lambda x: isinstance(x, Request),
         "item": is_item,
@@ -78,7 +78,7 @@ class ReturnsContract(Contract):
     def __init__(self, *args: Any, **kwargs: Any):
         super().__init__(*args, **kwargs)
 
-        if len(self.args) not in [1, 2, 3]:
+        if len(self.args) not in {1, 2, 3}:
             raise ValueError(
                 f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}"
             )

scrapy/core/downloader/handlers/datauri.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING
 
 from w3lib.url import parse_data_uri
 
@@ -17,9 +17,8 @@ class DataURIDownloadHandler(BaseDownloadHandler):
         uri = parse_data_uri(request.url)
         respcls = responsetypes.from_mimetype(uri.media_type)
 
-        resp_kwargs: dict[str, Any] = {}
         if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text":
             charset = uri.media_type_parameters.get("charset")
-            resp_kwargs["encoding"] = charset
+            return respcls(url=request.url, body=uri.data, encoding=charset)
 
-        return respcls(url=request.url, body=uri.data, **resp_kwargs)
+        return respcls(url=request.url, body=uri.data)

scrapy/core/downloader/handlers/file.py

@@ -7,6 +7,7 @@ from w3lib.url import file_uri_to_path
 
 from scrapy.core.downloader.handlers.base import BaseDownloadHandler
 from scrapy.responsetypes import responsetypes
+from scrapy.utils.asyncio import run_in_thread
 
 if TYPE_CHECKING:
     from scrapy import Request
@@ -16,6 +17,6 @@ if TYPE_CHECKING:
 class FileDownloadHandler(BaseDownloadHandler):
     async def download_request(self, request: Request) -> Response:
         filepath = file_uri_to_path(request.url)
-        body = Path(filepath).read_bytes()  # noqa: ASYNC240
+        body = await run_in_thread(Path(filepath).read_bytes)
         respcls = responsetypes.from_args(filename=filepath, body=body)
         return respcls(url=request.url, body=body)

scrapy/core/downloader/handlers/ftp.py

@@ -33,7 +33,7 @@ from __future__ import annotations
 import re
 from io import BytesIO
 from pathlib import Path
-from typing import TYPE_CHECKING, BinaryIO
+from typing import TYPE_CHECKING, BinaryIO, ClassVar
 from urllib.parse import unquote
 
 from twisted.internet.protocol import ClientCreator, Protocol
@@ -79,7 +79,7 @@ _CODE_RE = re.compile(r"\d+")
 
 
 class FTPDownloadHandler(BaseDownloadHandler):
-    CODE_MAPPING: dict[str, int] = {
+    CODE_MAPPING: ClassVar[dict[str, int]] = {
         "550": 404,
         "default": 503,
     }

scrapy/core/downloader/handlers/http11.py

@@ -147,7 +147,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
         # issue a callback after `_disconnect_timeout` seconds.
         #
         # See also https://github.com/scrapy/scrapy/issues/2653
-        delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, [])
+        delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, ())
 
         try:
             await maybe_deferred_to_future(d)

scrapy/core/http2/protocol.py

@@ -265,7 +265,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
     def _lose_connection_with_error(self, errors: list[BaseException]) -> None:
         """Helper function to lose the connection with the error sent as a
         reason"""
-        self._conn_lost_errors += errors
+        self._conn_lost_errors.extend(errors)
         assert self.transport is not None  # typing
         self.transport.loseConnection()
 
@@ -310,7 +310,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             if isinstance(e, FrameTooLargeError):
                 # hyper-h2 does not drop the connection in this scenario, we
                 # need to abort the connection manually.
-                self._conn_lost_errors += [e]
+                self._conn_lost_errors.append(e)
                 assert self.transport is not None  # typing
                 self.transport.abortConnection()
                 return

scrapy/core/http2/stream.py

@@ -22,6 +22,8 @@ from scrapy.utils._download_handlers import (
 from scrapy.utils.httpobj import urlparse_cached
 
 if TYPE_CHECKING:
+    from collections.abc import Sequence
+
     from hpack import HeaderTuple
 
     from scrapy.core.http2.protocol import H2ClientProtocol
@@ -150,7 +152,7 @@ class Stream:
             # flow control window
             "flow_controlled_size": 0,
             # Headers received after sending the request
-            "headers": Headers({}),
+            "headers": Headers(),
         }
 
         def _cancel(_: Any) -> None:
@@ -391,7 +393,7 @@ class Stream:
     def close(
         self,
         reason: StreamCloseReason,
-        errors: list[BaseException] | None = None,
+        errors: Sequence[BaseException] | None = None,
         from_protocol: bool = False,
     ) -> None:
         """Based on the reason sent we will handle each case."""
@@ -405,7 +407,7 @@ class Stream:
 
         # Have default value of errors as an empty list as
         # some cases can add a list of exceptions
-        errors = errors or []
+        errors = errors or ()
 
         if not from_protocol:
             self._protocol.pop_stream(self.stream_id)
@@ -470,7 +472,7 @@ class Stream:
             self._deferred_response.errback(ResponseFailed(errors))
 
         elif reason is StreamCloseReason.INACTIVE:
-            errors.insert(0, InactiveStreamClosed(self._request))
+            errors = (InactiveStreamClosed(self._request), *errors)
             self._deferred_response.errback(ResponseFailed(errors))
 
         else:

scrapy/core/scheduler.py

@@ -461,9 +461,10 @@ class Scheduler(BaseScheduler):
         except TypeError:  # pragma: no cover
             warn(
                 f"The __init__ method of {global_object_name(self.pqclass)} "
-                f"does not support a `start_queue_cls` keyword-only "
+                "does not support a `start_queue_cls` keyword-only "
-                f"parameter.",
+                "parameter.",
                 ScrapyDeprecationWarning,
+                stacklevel=2,
             )
             return build_from_crawler(
                 self.pqclass,
@@ -490,9 +491,10 @@ class Scheduler(BaseScheduler):
         except TypeError:  # pragma: no cover
             warn(
                 f"The __init__ method of {global_object_name(self.pqclass)} "
-                f"does not support a `start_queue_cls` keyword-only "
+                "does not support a `start_queue_cls` keyword-only "
-                f"parameter.",
+                "parameter.",
                 ScrapyDeprecationWarning,
+                stacklevel=2,
             )
             q = build_from_crawler(
                 self.pqclass,
@@ -521,7 +523,7 @@ class Scheduler(BaseScheduler):
     def _read_dqs_state(self, dqdir: str) -> Any:
         path = Path(dqdir, "active.json")
         if not path.exists():
-            return []
+            return ()
         with path.open(encoding="utf-8") as f:
             return json.load(f)
 

scrapy/core/spidermw.py

@@ -114,6 +114,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 f"Scrapy 2.13 for details: "
                 f"https://docs.scrapy.org/en/2.13/news.html",
                 ScrapyDeprecationWarning,
+                stacklevel=2,
             )
 
     def _add_middleware(self, mw: Any) -> None:
@@ -502,6 +503,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 f"copy-pasting. See the release notes of Scrapy 2.13 for "
                 f"details: https://docs.scrapy.org/en/2.13/news.html",
                 ScrapyDeprecationWarning,
+                stacklevel=2,
             )
 
         if (

scrapy/crawler.py

@@ -663,7 +663,7 @@ class CrawlerProcessBase(CrawlerRunnerBase):
         resolver_class = load_object(self.settings["DNS_RESOLVER"])
         # We pass self, which is CrawlerProcess, instead of Crawler here,
         # which works because the default resolvers only use crawler.settings.
-        resolver = build_from_crawler(resolver_class, self, reactor=reactor)  # type: ignore[arg-type]
+        resolver = build_from_crawler(resolver_class, self, reactor=reactor)  # type: ignore[call-overload]
         resolver.install_on_reactor()
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))

scrapy/downloadermiddlewares/cookies.py

@@ -139,7 +139,7 @@ class CookiesMiddleware:
         for key in ("name", "value", "path", "domain"):
             value = cookie.get(key)
             if value is None:
-                if key in ("name", "value"):
+                if key in {"name", "value"}:
                     msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
                     logger.warning(msg)
                     return None
@@ -176,7 +176,7 @@ class CookiesMiddleware:
         Extract cookies from the Request.cookies attribute
         """
         if not request.cookies:
-            return []
+            return ()
         cookies: Iterable[VerboseCookie]
         if isinstance(request.cookies, dict):
             cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items())

scrapy/downloadermiddlewares/httpcache.py

@@ -107,7 +107,7 @@ class HttpCacheMiddleware:
             return response
 
         # Skip cached responses and uncacheable requests
-        if "cached" in response.flags or "_dont_cache" in request.meta:
+        if "_dont_cache" in request.meta or "cached" in response.flags:
             request.meta.pop("_dont_cache", None)
             return response
 

scrapy/downloadermiddlewares/httpcompression.py

@@ -40,12 +40,13 @@ except ImportError:
     pass
 else:
     try:
-        brotli.Decompressor.can_accept_more_data
+        brotli.Decompressor.can_accept_more_data  # noqa: B018
     except AttributeError:  # pragma: no cover
         warnings.warn(
             "You have brotli installed. But 'br' encoding support now requires "
             "brotli's or brotlicffi's version >= 1.2.0. Please upgrade "
             "brotli/brotlicffi to make Scrapy decode 'br' encoded responses.",
+            stacklevel=2,
         )
     else:
         ACCEPTED_ENCODINGS.append(b"br")

scrapy/downloadermiddlewares/httpproxy.py

@@ -69,7 +69,7 @@ class HttpProxyMiddleware:
             _scheme = parsed.scheme
             if (
                 # 'no_proxy' is only supported by http schemes
-                _scheme not in ("http", "https")
+                _scheme not in {"http", "https"}
                 or (parsed.hostname and not proxy_bypass(parsed.hostname))
             ) and _scheme in self.proxies:
                 scheme = _scheme

scrapy/downloadermiddlewares/offsite.py

@@ -86,13 +86,13 @@ class OffsiteMiddleware:
                     "allowed_domains accepts only domains, not URLs. "
                     f"Ignoring URL entry {domain} in allowed_domains."
                 )
-                warnings.warn(message)
+                warnings.warn(message, stacklevel=2)
             elif port_pattern.search(domain):
                 message = (
                     "allowed_domains accepts only domains without ports. "
                     f"Ignoring entry {domain} in allowed_domains."
                 )
-                warnings.warn(message)
+                warnings.warn(message, stacklevel=2)
             else:
                 domains.append(re.escape(domain))
         regex = rf"^(.*\.)?({'|'.join(domains)})$"

scrapy/downloadermiddlewares/redirect.py

@@ -71,7 +71,7 @@ class BaseRedirectMiddleware:
             return
         redirect_cls = global_object_name(self.__class__)
         referer_cls = global_object_name(RefererMiddleware)
-        if self.__class__ in (RedirectMiddleware, MetaRefreshMiddleware):
+        if self.__class__ in {RedirectMiddleware, MetaRefreshMiddleware}:
             replacement = (
                 f"replace {redirect_cls} with a subclass that overrides the "
                 f"handle_referer() method"
@@ -208,14 +208,19 @@ class RedirectMiddleware(BaseRedirectMiddleware):
         if (
             request.meta.get("dont_redirect", False)
             or response.status
-            in getattr(self.crawler.spider, "handle_httpstatus_list", [])
+            in getattr(self.crawler.spider, "handle_httpstatus_list", ())
-            or response.status in request.meta.get("handle_httpstatus_list", [])
+            or response.status in request.meta.get("handle_httpstatus_list", ())
             or request.meta.get("handle_httpstatus_all", False)
         ):
             return response
 
-        allowed_status = (301, 302, 303, 307, 308)
+        if "Location" not in response.headers or response.status not in {
-        if "Location" not in response.headers or response.status not in allowed_status:
+            301,
+            302,
+            303,
+            307,
+            308,
+        }:
             return response
 
         assert response.headers["Location"] is not None
@@ -235,8 +240,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
         if urlparse_cached(redirected).scheme not in {"http", "https"}:
             return response
 
-        if (response.status in (301, 302) and request.method == "POST") or (
+        if (response.status in {301, 302} and request.method == "POST") or (
-            response.status == 303 and request.method not in ("GET", "HEAD")
+            response.status == 303 and request.method not in {"GET", "HEAD"}
         ):
             redirected = self._redirect_request_using_get(
                 request, response, redirected_url

scrapy/extensions/feedexport.py

@@ -200,7 +200,7 @@ class S3FeedStorage(BlockingFeedStorage):
         try:
             import boto3.session  # noqa: PLC0415
         except ImportError:
-            raise NotConfigured("missing boto3 library")
+            raise NotConfigured("missing boto3 library") from None
         u = urlparse(uri)
         assert u.hostname
         self.bucketname: str = u.hostname
@@ -250,10 +250,19 @@ class S3FeedStorage(BlockingFeedStorage):
 
     def _store_in_thread(self, file: IO[bytes]) -> None:
         file.seek(0)
-        kwargs: dict[str, Any] = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
+        if self.acl:
-        self.s3_client.upload_fileobj(
+            self.s3_client.upload_fileobj(
-            Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
+                Bucket=self.bucketname,
-        )
+                Key=self.keyname,
+                Fileobj=file,
+                ExtraArgs={"ACL": self.acl},
+            )
+        else:
+            self.s3_client.upload_fileobj(
+                Bucket=self.bucketname,
+                Key=self.keyname,
+                Fileobj=file,
+            )
         file.close()
 
 
@@ -468,9 +477,13 @@ class FeedExporter:
         # End: Backward compatibility for FEED_URI and FEED_FORMAT settings
 
         # 'FEEDS' setting takes precedence over 'FEED_URI'
-        for uri, feed_options in self.settings.getdict("FEEDS").items():
+        for settings_uri, feed_options in self.settings.getdict("FEEDS").items():
             # handle pathlib.Path objects
-            uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
+            uri = (
+                str(settings_uri)
+                if not isinstance(settings_uri, Path)
+                else settings_uri.absolute().as_uri()
+            )
             self.feeds[uri] = feed_complete_default_values_from_settings(
                 feed_options, self.settings
             )
@@ -692,9 +705,7 @@ class FeedExporter:
         uri_params_function: str | UriParamsCallableT | None,
         slot: FeedSlot | None = None,
     ) -> dict[str, Any]:
-        params = {}
+        params = {k: getattr(spider, k) for k in dir(spider)}
-        for k in dir(spider):
-            params[k] = getattr(spider, k)
         utc_now = datetime.now(tz=timezone.utc)
         params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-")
         params["batch_time"] = utc_now.isoformat().replace(":", "-")

scrapy/extensions/httpcache.py

@@ -106,10 +106,10 @@ class RFC2616Policy:
         if b"max-age" in cc or b"Expires" in response.headers:
             return True
         # Firefox fallbacks this statuses to one year expiration if none is set
-        if response.status in (300, 301, 308):
+        if response.status in {300, 301, 308}:
             return True
         # Other statuses without expiration requires at least one validator
-        if response.status in (200, 203, 401):
+        if response.status in {200, 203, 401}:
             return b"Last-Modified" in response.headers or b"ETag" in response.headers
         # Any other is probably not eligible for caching
         # Makes no sense to cache responses that does not contain expiration
@@ -216,7 +216,7 @@ class RFC2616Policy:
             return (date - lastmodified) / 10
 
         # This request can be cached indefinitely
-        if response.status in (300, 301, 308):
+        if response.status in {300, 301, 308}:
             return self.MAXAGE
 
         # Insufficient information to compute freshness lifetime

scrapy/extensions/memusage.py

@@ -39,8 +39,8 @@ class MemoryUsage:
         try:
             # stdlib's resource module is only available on unix platforms.
             self.resource = import_module("resource")
-        except ImportError:
+        except ImportError as exc:
-            raise NotConfigured
+            raise NotConfigured from exc
 
         self.crawler: Crawler = crawler
         self.warned: bool = False

scrapy/extensions/periodic_log.py

@@ -10,6 +10,7 @@ from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
 if TYPE_CHECKING:
+    from collections.abc import Sequence
     from json import JSONEncoder
 
     from twisted.internet.task import LoopingCall
@@ -31,8 +32,8 @@ class PeriodicLog:
         self,
         stats: StatsCollector,
         interval: float = 60.0,
-        ext_stats: dict[str, Any] = {},
+        ext_stats: dict[str, Any] | None = None,
-        ext_delta: dict[str, Any] = {},
+        ext_delta: dict[str, Any] | None = None,
         ext_timing_enabled: bool = False,
     ):
         self.stats: StatsCollector = stats
@@ -41,11 +42,19 @@ class PeriodicLog:
         self.task: AsyncioLoopingCall | LoopingCall | None = None
         self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
         self.ext_stats_enabled: bool = bool(ext_stats)
-        self.ext_stats_include: list[str] = ext_stats.get("include", [])
+        self.ext_stats_include: Sequence[str] = (
-        self.ext_stats_exclude: list[str] = ext_stats.get("exclude", [])
+            ext_stats.get("include", ()) if ext_stats else ()
+        )
+        self.ext_stats_exclude: Sequence[str] = (
+            ext_stats.get("exclude", ()) if ext_stats else ()
+        )
         self.ext_delta_enabled: bool = bool(ext_delta)
-        self.ext_delta_include: list[str] = ext_delta.get("include", [])
+        self.ext_delta_include: Sequence[str] = (
-        self.ext_delta_exclude: list[str] = ext_delta.get("exclude", [])
+            ext_delta.get("include", ()) if ext_delta else ()
+        )
+        self.ext_delta_exclude: Sequence[str] = (
+            ext_delta.get("exclude", ()) if ext_delta else ()
+        )
         self.ext_timing_enabled: bool = ext_timing_enabled
 
     @classmethod
@@ -143,7 +152,7 @@ class PeriodicLog:
         return {"stats": stats}
 
     def param_allowed(
-        self, stat_name: str, include: list[str], exclude: list[str]
+        self, stat_name: str, include: Sequence[str], exclude: Sequence[str]
     ) -> bool:
         if not include and not exclude:
             return True

scrapy/extensions/statsmailer.py

@@ -25,6 +25,7 @@ if TYPE_CHECKING:
 warnings.warn(
     "The scrapy.extensions.statsmailer module is deprecated and will be "
     "removed in a future release.",
+    stacklevel=2,
     category=ScrapyDeprecationWarning,
 )
 

scrapy/http/cookies.py

@@ -54,14 +54,14 @@ class CookieJar:
         if not IPV4_RE.search(req_host):
             hosts = potential_domain_matches(req_host)
             if "." not in req_host:
-                hosts += [req_host + ".local"]
+                hosts.append(req_host + ".local")
         else:
             hosts = [req_host]
 
         cookies = []
         for host in hosts:
             if host in self.jar._cookies:  # type: ignore[attr-defined]
-                cookies += self.jar._cookies_for_domain(host, wreq)  # type: ignore[attr-defined]
+                cookies.extend(self.jar._cookies_for_domain(host, wreq))  # type: ignore[attr-defined]
 
         attrs = self.jar._cookie_attrs(cookies)  # type: ignore[attr-defined]
         if attrs and not wreq.has_header("Cookie"):

scrapy/http/request/form.py

@@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst
 from __future__ import annotations
 
 from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any, TypeAlias, cast
+from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
 
 from parsel.csstranslator import HTMLTranslator
@@ -39,7 +39,7 @@ FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None
 class FormRequest(Request):
     __slots__ = ()
 
-    valid_form_methods = ["GET", "POST"]
+    valid_form_methods: ClassVar[list[str]] = ["GET", "POST"]
 
     def __init__(
         self, *args: Any, formdata: FormdataType = None, **kwargs: Any
@@ -153,7 +153,7 @@ def _get_form(
     try:
         form = forms[formnumber]
     except IndexError:
-        raise IndexError(f"Form number {formnumber} not found in {response}")
+        raise IndexError(f"Form number {formnumber} not found in {response}") from None
     return cast("FormElement", form)
 
 
@@ -167,7 +167,7 @@ def _get_inputs(
     try:
         formdata_keys = dict(formdata or ()).keys()
     except (ValueError, TypeError):
-        raise ValueError("formdata should be a dict or iterable of tuples")
+        raise ValueError("formdata should be a dict or iterable of tuples") from None
 
     if not formdata:
         formdata = []

scrapy/http/request/json_request.py

@@ -36,7 +36,9 @@ class JsonRequest(Request):
         data_passed: bool = data is not None
 
         if body_passed and data_passed:
-            warnings.warn("Both body and data passed. data will be ignored")
+            warnings.warn(
+                "Both body and data passed. data will be ignored", stacklevel=2
+            )
         elif not body_passed and data_passed:
             kwargs["body"] = self._dumps(data)
             if "method" not in kwargs:
@@ -68,7 +70,9 @@ class JsonRequest(Request):
         data_passed: bool = data is not None
 
         if body_passed and data_passed:
-            warnings.warn("Both body and data passed. data will be ignored")
+            warnings.warn(
+                "Both body and data passed. data will be ignored", stacklevel=2
+            )
         elif not body_passed and data_passed:
             kwargs["body"] = self._dumps(data)
 

scrapy/http/response/__init__.py

@@ -86,7 +86,7 @@ class Response(object_ref):
             raise AttributeError(
                 "Response.cb_kwargs not available, this response "
                 "is not tied to any request"
-            )
+            ) from None
 
     @property
     def meta(self) -> dict[str, Any]:
@@ -95,7 +95,7 @@ class Response(object_ref):
         except AttributeError:
             raise AttributeError(
                 "Response.meta not available, this response is not tied to any request"
-            )
+            ) from None
 
     @property
     def url(self) -> str:

scrapy/http/response/text.py

@@ -297,7 +297,7 @@ def _url_from_selector(sel: parsel.Selector) -> str:
         return strip_html5_whitespace(sel.root)
     if not hasattr(sel.root, "tag"):
         raise _InvalidSelector(f"Unsupported selector: {sel}")
-    if sel.root.tag not in ("a", "link"):
+    if sel.root.tag not in {"a", "link"}:
         raise _InvalidSelector(
             f"Only <a> and <link> elements are supported; got <{sel.root.tag}>"
         )

scrapy/linkextractors/lxmlhtml.py

@@ -114,8 +114,8 @@ class LxmlParserLinkExtractor:
             # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
             try:
                 if self.strip:
-                    attr_val = strip_html5_whitespace(attr_val)
+                    attr_val = strip_html5_whitespace(attr_val)  # noqa: PLW2901 this is intended
-                attr_val = urljoin(base_url, attr_val)
+                attr_val = urljoin(base_url, attr_val)  # noqa: PLW2901
             except ValueError:
                 continue  # skipping bogus links
             else:
@@ -245,7 +245,7 @@ class LxmlLinkExtractor:
             if self.allow_res
             else [True]
         )
-        denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else []
+        denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else ()
         return any(allowed) and not any(denied)
 
     def _process_links(self, links: list[Link]) -> list[Link]:

scrapy/mail.py

@@ -55,6 +55,7 @@ def _to_bytes_or_none(text: str | bytes | None) -> bytes | None:
 warnings.warn(
     "The scrapy.mail module is deprecated and will be removed in a future release. "
     "Please use a dedicated Python mail library instead.",
+    stacklevel=2,
     category=ScrapyDeprecationWarning,
 )
 

scrapy/pipelines/files.py

@@ -18,7 +18,7 @@ from contextlib import suppress
 from ftplib import FTP
 from io import BytesIO
 from pathlib import Path
-from typing import IO, TYPE_CHECKING, Any, NoReturn, Protocol, TypedDict, cast
+from typing import IO, TYPE_CHECKING, Any, ClassVar, NoReturn, Protocol, TypedDict, cast
 from urllib.parse import urlparse
 
 from itemadapter import ItemAdapter
@@ -161,7 +161,7 @@ class S3FilesStore:
     AWS_VERIFY = None
 
     POLICY = "private"  # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler()
-    HEADERS = {
+    HEADERS: ClassVar[dict[str, str]] = {
         "Cache-Control": "max-age=172800",
     }
 
@@ -226,7 +226,7 @@ class S3FilesStore:
                 Bucket=self.bucket,
                 Key=key_name,
                 Body=buf,
-                Metadata={k: str(v) for k, v in (meta or {}).items()},
+                Metadata={k: str(v) for k, v in meta.items()} if meta else {},
                 ACL=self.POLICY,
                 **extra,
             )
@@ -269,7 +269,9 @@ class S3FilesStore:
             try:
                 kwarg = mapping[key]
             except KeyError:
-                raise TypeError(f'Header "{key}" is not supported by botocore')
+                raise TypeError(
+                    f'Header "{key}" is not supported by botocore'
+                ) from None
             extra[kwarg] = value
         return extra
 
@@ -339,7 +341,7 @@ class GCSFilesStore:
         blob_path = self._get_blob_path(path)
         blob = self.bucket.blob(blob_path)
         blob.cache_control = self.CACHE_CONTROL
-        blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
+        blob.metadata = {k: str(v) for k, v in meta.items()} if meta else {}
         return deferred_from_coro(
             run_in_thread(
                 blob.upload_from_string,
@@ -435,7 +437,7 @@ class FilesPipeline(MediaPipeline):
 
     MEDIA_NAME: str = "file"
     EXPIRES: int = 90
-    STORE_SCHEMES: dict[str, type[FilesStoreProtocol]] = {
+    STORE_SCHEMES: ClassVar[dict[str, type[FilesStoreProtocol]]] = {
         "": FSFilesStore,
         "file": FSFilesStore,
         "s3": S3FilesStore,
@@ -656,7 +658,7 @@ class FilesPipeline(MediaPipeline):
                 exc_info=True,
                 extra={"spider": info.spider},
             )
-            raise FileException(str(exc))
+            raise FileException(str(exc)) from exc
 
         return {
             "url": request.url,

scrapy/pipelines/images.py

@@ -11,7 +11,7 @@ import hashlib
 import warnings
 from contextlib import suppress
 from io import BytesIO
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, ClassVar
 
 from itemadapter import ItemAdapter
 
@@ -49,7 +49,7 @@ class ImagesPipeline(FilesPipeline):
     MIN_WIDTH: int = 0
     MIN_HEIGHT: int = 0
     EXPIRES: int = 90
-    THUMBS: dict[str, tuple[int, int]] = {}
+    THUMBS: ClassVar[dict[str, tuple[int, int]]] = {}
     DEFAULT_IMAGES_URLS_FIELD = "image_urls"
     DEFAULT_IMAGES_RESULT_FIELD = "images"
 
@@ -76,7 +76,7 @@ class ImagesPipeline(FilesPipeline):
         except ImportError:
             raise NotConfigured(
                 "ImagesPipeline requires installing Pillow 8.3.2 or later"
-            )
+            ) from None
 
         super().__init__(store_uri, crawler=crawler)
 
@@ -191,7 +191,7 @@ class ImagesPipeline(FilesPipeline):
         *,
         response_body: BytesIO,
     ) -> tuple[Image.Image, BytesIO]:
-        if image.format in ("PNG", "WEBP") and image.mode == "RGBA":
+        if image.format in {"PNG", "WEBP"} and image.mode == "RGBA":
             background = self._Image.new("RGBA", image.size, (255, 255, 255))
             background.paste(image, image)
             image = background.convert("RGB")

scrapy/pipelines/media.py

@@ -230,9 +230,9 @@ class MediaPipeline(ABC):
         if isinstance(result, Failure):
             # minimize cached information for failure
             result.cleanFailure()
-            result.frames = []
+            result.frames.clear()
             if TWISTED_FAILURE_HAS_STACK:
-                result.stack = []  # type: ignore[method-assign]
+                result.stack.clear()
             # This code fixes a memory leak by avoiding to keep references to
             # the Request and Response objects on the Media Pipeline cache.
             #

scrapy/pqueues.py

@@ -350,8 +350,9 @@ class DownloaderAwarePriorityQueue:
         self.crawler: Crawler = crawler
 
         self.pqueues: dict[str, ScrapyPriorityQueue] = {}  # slot -> priority queue
-        for slot, startprios in (slot_startprios or {}).items():
+        if slot_startprios:
-            self.pqueues[slot] = self.pqfactory(slot, startprios)
+            for slot, startprios in slot_startprios.items():
+                self.pqueues[slot] = self.pqfactory(slot, startprios)
 
     def pqfactory(
         self, slot: str, startprios: Iterable[int] = ()

scrapy/responsetypes.py

@@ -8,7 +8,7 @@ from __future__ import annotations
 from io import StringIO
 from mimetypes import MimeTypes
 from pkgutil import get_data
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, ClassVar
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
@@ -19,7 +19,7 @@ if TYPE_CHECKING:
 
 
 class ResponseTypes:
-    CLASSES = {
+    CLASSES: ClassVar[dict[str, str]] = {
         "text/html": "scrapy.http.HtmlResponse",
         "application/atom+xml": "scrapy.http.XmlResponse",
         "application/rdf+xml": "scrapy.http.XmlResponse",

scrapy/settings/__init__.py

@@ -184,15 +184,15 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
         try:
             return bool(int(got))
         except ValueError:
-            if got in ("True", "true"):
+            if got in {"True", "true"}:
                 return True
-            if got in ("False", "false"):
+            if got in {"False", "false"}:
                 return False
             raise ValueError(
                 "Supported values for boolean settings "
                 "are 0/1, True/False, '0'/'1', "
                 "'True'/'False' and 'true'/'false'"
-            )
+            ) from None
 
     def getint(self, name: _SettingsKey, default: int = 0) -> int:
         """

scrapy/spiderloader.py

@@ -77,6 +77,7 @@ class SpiderLoader:
             warnings.warn(
                 "There are several spiders with the same name:\n\n"
                 f"{dupes_string}\n\n  This can cause unexpected behavior.",
+                stacklevel=2,
                 category=UserWarning,
             )
 
@@ -96,6 +97,7 @@ class SpiderLoader:
                         f"\n{traceback.format_exc()}Could not load spiders "
                         f"from module '{name}'. "
                         "See above traceback for details.",
+                        stacklevel=2,
                         category=RuntimeWarning,
                     )
                 else:
@@ -114,7 +116,7 @@ class SpiderLoader:
         try:
             return self._spiders[spider_name]
         except KeyError:
-            raise KeyError(f"Spider not found: {spider_name}")
+            raise KeyError(f"Spider not found: {spider_name}") from None
 
     def find_by_request(self, request: Request) -> list[str]:
         """

scrapy/spidermiddlewares/httperror.py

@@ -88,5 +88,5 @@ class HttpErrorMiddleware:
                 {"response": response},
                 extra={"spider": self.crawler.spider},
             )
-            return []
+            return ()
         return None

scrapy/spidermiddlewares/referer.py

@@ -103,7 +103,7 @@ class ReferrerPolicy(ABC):
         return self.tls_protected(url)
 
     def tls_protected(self, url: str) -> bool:
-        return urlparse(url).scheme in ("https", "ftps")
+        return urlparse(url).scheme in {"https", "ftps"}
 
 
 class NoReferrerPolicy(ReferrerPolicy):
@@ -424,7 +424,7 @@ class RefererMiddleware(BaseSpiderMiddleware):
             msg += " (import paths from the response Referrer-Policy header are not allowed)"
         if not warning_only:
             raise RuntimeError(msg)
-        warnings.warn(msg, RuntimeWarning)
+        warnings.warn(msg, RuntimeWarning, stacklevel=2)
         return None
 
     def get_processed_request(

scrapy/spiders/crawl.py

@@ -102,11 +102,12 @@ class CrawlSpider(Spider):
         self._compile_rules()
         if method_is_overridden(self.__class__, CrawlSpider, "_parse_response"):
             warnings.warn(
-                f"The CrawlSpider._parse_response method, which the "
+                "The CrawlSpider._parse_response method, which the "
                 f"{global_object_name(self.__class__)} class overrides, is "
-                f"deprecated: it will be removed in future Scrapy releases. "
+                "deprecated: it will be removed in future Scrapy releases. "
-                f"Please override the CrawlSpider.parse_with_rules method "
+                "Please override the CrawlSpider.parse_with_rules method "
-                f"instead."
+                "instead.",
+                stacklevel=2,
             )
 
     def _parse(self, response: Response, **kwargs: Any) -> Any:
@@ -118,7 +119,7 @@ class CrawlSpider(Spider):
         )
 
     def parse_start_url(self, response: Response, **kwargs: Any) -> Any:
-        return []
+        return ()
 
     def process_results(
         self, response: Response, results: Iterable[Any]
@@ -209,8 +210,9 @@ class CrawlSpider(Spider):
     def _compile_rules(self) -> None:
         self._rules = []
         for rule in self.rules:
-            self._rules.append(copy.copy(rule))
+            copied_rule = copy.copy(rule)
-            self._rules[-1]._compile(self)
+            copied_rule._compile(self)
+            self._rules.append(copied_rule)
 
     @classmethod
     def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:

scrapy/spiders/sitemap.py

@@ -49,7 +49,7 @@ class SitemapSpider(Spider):
         self._cbs: list[tuple[re.Pattern[str], CallbackT]] = []
         for r, c in self.sitemap_rules:
             if isinstance(c, str):
-                c = cast("CallbackT", getattr(self, c))
+                c = cast("CallbackT", getattr(self, c))  # noqa: PLW2901
             self._cbs.append((regex(r), c))
         self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]
 

scrapy/utils/console.py

@@ -13,7 +13,7 @@ KnownShellsT = dict[str, Callable[..., EmbedFuncT]]
 
 
 def _embed_ipython_shell(
-    namespace: dict[str, Any] = {}, banner: str = ""
+    namespace: dict[str, Any] | None = None, banner: str = ""
 ) -> EmbedFuncT:
     """Start an IPython Shell"""
     try:
@@ -28,7 +28,7 @@ def _embed_ipython_shell(
         )
 
     @wraps(_embed_ipython_shell)
-    def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
+    def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
         config = load_default_config()  # type: ignore[no-untyped-call]
         # Always use .instance() to ensure _instance propagation to all parents
         # this is needed for <TAB> completion works well for new imports
@@ -44,26 +44,26 @@ def _embed_ipython_shell(
 
 
 def _embed_bpython_shell(
-    namespace: dict[str, Any] = {}, banner: str = ""
+    namespace: dict[str, Any] | None = None, banner: str = ""
 ) -> EmbedFuncT:
     """Start a bpython shell"""
     import bpython  # noqa: PLC0415
 
     @wraps(_embed_bpython_shell)
-    def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
+    def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
         bpython.embed(locals_=namespace, banner=banner)
 
     return wrapper
 
 
 def _embed_ptpython_shell(
-    namespace: dict[str, Any] = {}, banner: str = ""
+    namespace: dict[str, Any] | None = None, banner: str = ""
 ) -> EmbedFuncT:
     """Start a ptpython shell"""
     import ptpython.repl  # noqa: PLC0415  # pylint: disable=import-error
 
     @wraps(_embed_ptpython_shell)
-    def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
+    def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
         print(banner)
         ptpython.repl.embed(locals=namespace)
 
@@ -71,7 +71,7 @@ def _embed_ptpython_shell(
 
 
 def _embed_standard_shell(
-    namespace: dict[str, Any] = {}, banner: str = ""
+    namespace: dict[str, Any] | None = None, banner: str = ""
 ) -> EmbedFuncT:
     """Start a standard python shell"""
     try:  # readline module is only available on unix systems
@@ -84,7 +84,7 @@ def _embed_standard_shell(
         readline.parse_and_bind("tab:complete")  # type: ignore[attr-defined,unused-ignore]
 
     @wraps(_embed_standard_shell)
-    def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
+    def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
         code.interact(banner=banner, local=namespace)
 
     return wrapper

scrapy/utils/curl.py

@@ -106,7 +106,7 @@ def curl_to_request_kwargs(
     if argv:
         msg = f"Unrecognized options: {', '.join(argv)}"
         if ignore_unknown_options:
-            warnings.warn(msg)
+            warnings.warn(msg, stacklevel=2)
         else:
             raise ValueError(msg)
 

scrapy/utils/defer.py

@@ -8,6 +8,7 @@ import asyncio
 import inspect
 import warnings
 from asyncio import Future
+from collections import deque
 from collections.abc import Awaitable, Coroutine, Iterable, Iterator
 from functools import wraps
 from typing import (
@@ -230,7 +231,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
         self.callable_args: tuple[Any, ...] = callable_args
         self.callable_kwargs: dict[str, Any] = callable_kwargs
         self.finished: bool = False
-        self.waiting_deferreds: list[Deferred[Any]] = []
+        self.waiting_deferreds: deque[Deferred[Any]] = deque()
         self.anext_deferred: Deferred[_T] | None = None
 
     def _callback(self, result: _T) -> None:
@@ -241,7 +242,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
         callable_result = self.callable(
             result, *self.callable_args, **self.callable_kwargs
         )
-        d = self.waiting_deferreds.pop(0)
+        d = self.waiting_deferreds.popleft()
         if isinstance(callable_result, Deferred):
             callable_result.chainDeferred(d)
         else:

scrapy/utils/deprecate.py

@@ -130,7 +130,7 @@ def create_deprecated_class(
         # deprecated class is in jinja2 template). __module__ attribute is not
         # important enough to raise an exception as users may be unable
         # to fix inspect.stack() errors.
-        warnings.warn(f"Error detecting parent module: {e!r}")
+        warnings.warn(f"Error detecting parent module: {e!r}", stacklevel=2)
 
     return deprecated_cls
 
@@ -160,6 +160,7 @@ def update_classpath(path: Any) -> Any:
             warnings.warn(
                 f"`{path}` class is deprecated, use `{new_path}` instead",
                 ScrapyDeprecationWarning,
+                stacklevel=2,
             )
             return new_path
     return path

scrapy/utils/misc.py

@@ -13,7 +13,7 @@ from contextlib import contextmanager
 from functools import partial
 from importlib import import_module
 from pkgutil import iter_modules
-from typing import IO, TYPE_CHECKING, Any, TypeVar, cast
+from typing import IO, TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, overload
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import Item
@@ -28,9 +28,20 @@ if TYPE_CHECKING:
 
 
 _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
-T = TypeVar("T")
+_ITER_T = TypeVar("_ITER_T", bound=dict | Item | str | bytes)
+_T = TypeVar("_T")
+_T_co = TypeVar("_T_co", covariant=True)
+_P = ParamSpec("_P")
 
 
+@overload
+def arg_to_iter(arg: None) -> tuple[()]: ...
+@overload
+def arg_to_iter(arg: _ITER_T) -> Iterable[_ITER_T]: ...
+@overload
+def arg_to_iter(arg: Iterable[_T]) -> Iterable[_T]: ...
+@overload
+def arg_to_iter(arg: _T) -> Iterable[_T]: ...
 def arg_to_iter(arg: Any) -> Iterable[Any]:
     """Convert an argument to an iterable. The argument can be a None, single
     value, or an iterable.
@@ -38,9 +49,9 @@ def arg_to_iter(arg: Any) -> Iterable[Any]:
     Exception: if arg is a dict, [arg] will be returned
     """
     if arg is None:
-        return []
+        return ()
     if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):
-        return cast("Iterable[Any]", arg)
+        return arg
     return [arg]
 
 
@@ -64,7 +75,7 @@ def load_object(path: str | Callable[..., Any]) -> Any:
     try:
         dot = path.rindex(".")
     except ValueError:
-        raise ValueError(f"Error loading object '{path}': not a full path")
+        raise ValueError(f"Error loading object '{path}': not a full path") from None
 
     module, name = path[:dot], path[dot + 1 :]
     mod = import_module(module)
@@ -72,7 +83,9 @@ def load_object(path: str | Callable[..., Any]) -> Any:
     try:
         obj = getattr(mod, name)
     except AttributeError:
-        raise NameError(f"Module '{module}' doesn't define any object named '{name}'")
+        raise NameError(
+            f"Module '{module}' doesn't define any object named '{name}'"
+        ) from None
 
     return obj
 
@@ -152,9 +165,40 @@ def rel_has_nofollow(rel: str | None) -> bool:
     return rel is not None and "nofollow" in rel.replace(",", " ").split()
 
 
+class SupportsFromCrawler(Protocol[_T_co, _P]):
+    @classmethod
+    def from_crawler(
+        cls, crawler: Crawler, /, *args: _P.args, **kwargs: _P.kwargs
+    ) -> _T_co: ...
+
+
+@overload
 def build_from_crawler(
-    objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any
+    objcls: SupportsFromCrawler[_T_co, _P],
-) -> T:
+    crawler: Crawler,
+    /,
+    *args: _P.args,
+    **kwargs: _P.kwargs,
+) -> _T_co: ...
+
+
+@overload
+def build_from_crawler(
+    objcls: Callable[_P, _T_co],
+    crawler: Crawler,
+    /,
+    *args: _P.args,
+    **kwargs: _P.kwargs,
+) -> _T_co: ...
+
+
+def build_from_crawler(
+    objcls: Any,
+    crawler: Crawler,
+    /,
+    *args: Any,
+    **kwargs: Any,
+) -> Any:
     """Construct a class instance using its ``from_crawler()`` or ``__init__()`` constructor.
 
     .. versionadded:: 2.12
@@ -164,14 +208,14 @@ def build_from_crawler(
     Raises ``TypeError`` if the resulting instance is ``None``.
     """
     if hasattr(objcls, "from_crawler"):
-        instance = objcls.from_crawler(crawler, *args, **kwargs)  # type: ignore[attr-defined]
+        instance = objcls.from_crawler(crawler, *args, **kwargs)
         method_name = "from_crawler"
     else:
         instance = objcls(*args, **kwargs)
         method_name = "__new__"
     if instance is None:
         raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
-    return cast("T", instance)
+    return instance
 
 
 @contextmanager

scrapy/utils/project.py

@@ -20,7 +20,8 @@ def inside_project() -> bool:
             import_module(scrapy_module)
         except ImportError as exc:
             warnings.warn(
-                f"Cannot import scrapy settings module {scrapy_module}: {exc}"
+                f"Cannot import scrapy settings module {scrapy_module}: {exc}",
+                stacklevel=2,
             )
         else:
             return True

scrapy/utils/reactor.py

@@ -76,7 +76,7 @@ class CallLaterOnce(Generic[_T]):
 
         for d in self._deferreds:
             call_later(0, d.callback, None)
-        self._deferreds = []
+        self._deferreds.clear()
 
         return result
 

scrapy/utils/request.py

@@ -169,7 +169,7 @@ def _get_method(obj: Any, name: Any) -> Any:
     try:
         return getattr(obj, name)
     except AttributeError:
-        raise ValueError(f"Method {name!r} not found in: {obj}")
+        raise ValueError(f"Method {name!r} not found in: {obj}") from None
 
 
 def request_to_curl(request: Request) -> str:

scrapy/utils/testproc.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 import os
 import sys
 import warnings
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING, ClassVar, cast
 
 from twisted.internet.defer import Deferred
 from twisted.internet.protocol import ProcessProtocol
@@ -21,12 +21,13 @@ if TYPE_CHECKING:
 warnings.warn(
     "The scrapy.utils.testproc module is deprecated.",
     ScrapyDeprecationWarning,
+    stacklevel=2,
 )
 
 
 class ProcessTest:
     command: str | None = None
-    prefix = [sys.executable, "-m", "scrapy.cmdline"]
+    prefix: ClassVar[list[str]] = [sys.executable, "-m", "scrapy.cmdline"]
     cwd = os.getcwd()  # trial chdirs to temp dir  # noqa: PTH109
 
     def execute(

scrapy/utils/testsite.py

@@ -9,6 +9,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 warnings.warn(
     "The scrapy.utils.testsite module is deprecated.",
     ScrapyDeprecationWarning,
+    stacklevel=2,
 )
 
 

scrapy/utils/url.py

@@ -19,13 +19,18 @@ from w3lib.url import parse_url as _parse_url
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
+_DEPRECATED_NAMES: frozenset[str] = frozenset(
+    {"_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects}
+)
+
 
 def __getattr__(name: str) -> Any:
-    if name in ("_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects):
+    if name in _DEPRECATED_NAMES:
         obj_type = "attribute" if name == "_safe_chars" else "function"
         warnings.warn(
             f"The scrapy.utils.url.{name} {obj_type} is deprecated, use w3lib.url.{name} instead.",
             ScrapyDeprecationWarning,
+            stacklevel=2,
         )
         return getattr(import_module("w3lib.url"), name)
 
@@ -45,15 +50,18 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:
     host = _parse_url(url).netloc.lower()
     if not host:
         return False
-    domains = [d.lower() for d in domains]
+    return any((host == d) or (host.endswith(f".{d}")) for d in map(str.lower, domains))
-    return any((host == d) or (host.endswith(f".{d}")) for d in domains)
+
+
+def _spider_domains(spider: type[Spider]) -> Iterable[str]:
+    yield spider.name
+    if allowed_domains := getattr(spider, "allowed_domains", None):
+        yield from allowed_domains
 
 
 def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool:
     """Return True if the url belongs to the given spider"""
-    return url_is_from_any_domain(
+    return url_is_from_any_domain(url, _spider_domains(spider))
-        url, [spider.name, *getattr(spider, "allowed_domains", [])]
-    )
 
 
 def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:
@@ -184,11 +192,11 @@ def strip_url(
         strip_default_port
         and parsed_url.port
         and (parsed_url.scheme, parsed_url.port)
-        in (
+        in {
             ("http", 80),
             ("https", 443),
             ("ftp", 21),
-        )
+        }
     ):
         netloc = netloc.replace(f":{parsed_url.port}", "")
 

tests/test_spidermiddleware_httperror.py

@@ -88,7 +88,7 @@ class TestHttpErrorMiddleware:
     def test_process_spider_exception(
         self, mw: HttpErrorMiddleware, res404: Response
     ) -> None:
-        assert mw.process_spider_exception(res404, HttpError(res404)) == []
+        assert mw.process_spider_exception(res404, HttpError(res404)) == ()
         assert mw.process_spider_exception(res404, Exception()) is None
 
     def test_handle_httpstatus_list(