diff --git a/pyproject.toml b/pyproject.toml index b343cce60..117edce0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -415,25 +415,6 @@ ignore = [ "SIM115", # Yoda condition detected "SIM300", - - # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") - - # Assigning to `os.environ` doesn't clear the environment. - "B003", - # Do not use mutable data structures for argument defaults. - "B006", - # Found useless expression. - "B018", - # No explicit stacklevel argument found. - "B028", - # Within an `except` clause, raise exceptions with `raise ... from` - "B904", - # `for` loop variable overwritten by assignment target - "PLW2901", - # Mutable class attributes should be annotated with `typing.ClassVar` - "RUF012", - # Use capitalized environment variable - "SIM112", ] [tool.ruff.lint.flake8-tidy-imports] @@ -453,8 +434,28 @@ split-on-trailing-comma = false "scrapy/linkextractors/__init__.py" = ["E402"] "scrapy/spiders/__init__.py" = ["E402"] -# Skip bandit and allow blocking file I/O in tests -"tests/**" = ["ASYNC240", "S"] +"tests/**" = [ + # Skip bandit and allow blocking file I/O in tests + "ASYNC240", + "S", + # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") + # Assigning to `os.environ` doesn't clear the environment. + "B003", + # Do not use mutable data structures for argument defaults. + "B006", + # Found useless expression. + "B018", + # No explicit stacklevel argument found. + "B028", + # Within an `except` clause, raise exceptions with `raise ... from` + "B904", + # `for` loop variable overwritten by assignment target + "PLW2901", + # Mutable class attributes should be annotated with `typing.ClassVar` + "RUF012", + # Use capitalized environment variable + "SIM112", +] # Issues pending a review: "docs/conf.py" = ["E402"] diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py index 057ddba4d..8aadc90e7 100644 --- a/scrapy/cmdline.py +++ b/scrapy/cmdline.py @@ -46,7 +46,7 @@ def _iter_command_classes(module_name: str) -> Iterable[type[ScrapyCommand]]: inspect.isclass(obj) and issubclass(obj, ScrapyCommand) and obj.__module__ == module.__name__ - and obj not in (ScrapyCommand, BaseRunSpiderCommand) + and obj not in {ScrapyCommand, BaseRunSpiderCommand} ): yield obj diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index d666ca796..e213d0d94 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -10,7 +10,7 @@ import os import warnings from abc import ABC, abstractmethod from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from twisted.python import failure @@ -30,7 +30,7 @@ class ScrapyCommand(ABC): crawler_process: CrawlerProcessBase | None = None # set in scrapy.cmdline # default settings to be used for this command instead of global defaults - default_settings: dict[str, Any] = {} + default_settings: ClassVar[dict[str, Any]] = {} exitcode: int = 0 @@ -115,7 +115,9 @@ class ScrapyCommand(ABC): try: self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline") except ValueError: - raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False) + raise UsageError( + "Invalid -s value, use -s NAME=VALUE", print_help=False + ) from None if opts.logfile: self.settings.set("LOG_ENABLED", True, priority="cmdline") @@ -181,7 +183,9 @@ class BaseRunSpiderCommand(ScrapyCommand): try: opts.spargs = arglist_to_dict(opts.spargs) except ValueError: - raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False) + raise UsageError( + "Invalid -a value, use -a NAME=VALUE", print_help=False + ) from None if opts.output or opts.overwrite_output: assert self.settings is not None feeds = feed_process_params_from_cli( diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index cc39d344a..0a5b431a2 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -3,7 +3,7 @@ from __future__ import annotations import subprocess import sys import time -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from urllib.parse import urlencode import scrapy @@ -18,7 +18,7 @@ if TYPE_CHECKING: class Command(ScrapyCommand): - default_settings = { + default_settings: ClassVar[dict[str, Any]] = { "LOG_LEVEL": "INFO", "LOGSTATS_INTERVAL": 1, "CLOSESPIDER_TIMEOUT": 10, diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 17e66e20c..1e4e09135 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -2,7 +2,7 @@ import argparse import time from collections import defaultdict from collections.abc import AsyncIterator -from typing import Any +from typing import Any, ClassVar from unittest import TextTestResult as _TextTestResult from unittest import TextTestRunner @@ -44,7 +44,7 @@ class TextTestResult(_TextTestResult): class Command(ScrapyCommand): requires_project = True - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def syntax(self) -> str: return "[options] " diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py index f2d52673a..cd7c57f28 100644 --- a/scrapy/commands/edit.py +++ b/scrapy/commands/edit.py @@ -1,6 +1,7 @@ import argparse import os import sys +from typing import Any, ClassVar from scrapy.commands import ScrapyCommand from scrapy.exceptions import UsageError @@ -10,7 +11,7 @@ from scrapy.spiderloader import get_spider_loader class Command(ScrapyCommand): requires_project = True requires_crawler_process = False - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def syntax(self) -> str: return "" diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 9d2742afd..b030be4de 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -5,7 +5,7 @@ import shutil import string from importlib import import_module from pathlib import Path -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, ClassVar, cast from urllib.parse import urlparse import scrapy @@ -47,7 +47,7 @@ def verify_url_scheme(url: str) -> str: class Command(ScrapyCommand): requires_crawler_process = False - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def syntax(self) -> str: return "[options] " diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py index b4dc97f3d..ad55dba66 100644 --- a/scrapy/commands/list.py +++ b/scrapy/commands/list.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, ClassVar from scrapy.commands import ScrapyCommand from scrapy.spiderloader import get_spider_loader @@ -12,7 +12,7 @@ if TYPE_CHECKING: class Command(ScrapyCommand): requires_project = True requires_crawler_process = False - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def short_desc(self) -> str: return "List available spiders" diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index e1e027c95..632186a7d 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -4,7 +4,7 @@ import functools import inspect import json import logging -from typing import TYPE_CHECKING, Any, TypeVar, overload +from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, overload from itemadapter import ItemAdapter from twisted.internet.defer import Deferred, maybeDeferred @@ -39,8 +39,8 @@ class Command(BaseRunSpiderCommand): requires_project = True spider: Spider | None = None - items: dict[int, list[Any]] = {} - requests: dict[int, list[Request]] = {} + items: ClassVar[dict[int, list[Any]]] = {} + requests: ClassVar[dict[int, list[Request]]] = {} spidercls: type[Spider] | None first_response = None @@ -387,7 +387,7 @@ class Command(BaseRunSpiderCommand): "Invalid -m/--meta value, pass a valid json string to -m or --meta. " 'Example: --meta=\'{"foo" : "bar"}\'', print_help=False, - ) + ) from None def process_request_cb_kwargs(self, opts: argparse.Namespace) -> None: if opts.cbkwargs: @@ -398,7 +398,7 @@ class Command(BaseRunSpiderCommand): "Invalid --cbkwargs value, pass a valid json string to --cbkwargs. " 'Example: --cbkwargs=\'{"foo" : "bar"}\'', print_help=False, - ) + ) from None def run(self, args: list[str], opts: argparse.Namespace) -> None: # parse arguments diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py index eeb1303e2..0b9036457 100644 --- a/scrapy/commands/runspider.py +++ b/scrapy/commands/runspider.py @@ -3,7 +3,7 @@ from __future__ import annotations import sys from importlib import import_module from pathlib import Path -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, ClassVar from scrapy.commands import BaseRunSpiderCommand from scrapy.exceptions import UsageError @@ -18,7 +18,7 @@ if TYPE_CHECKING: def _import_file(filepath: str | PathLike[str]) -> ModuleType: abspath = Path(filepath).resolve() - if abspath.suffix not in (".py", ".pyw"): + if abspath.suffix not in {".py", ".pyw"}: raise ValueError(f"Not a Python source file: {abspath}") dirname = str(abspath.parent) sys.path = [dirname, *sys.path] @@ -30,7 +30,9 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType: class Command(BaseRunSpiderCommand): - default_settings = {"SPIDER_LOADER_CLASS": DummySpiderLoader} + default_settings: ClassVar[dict[str, Any]] = { + "SPIDER_LOADER_CLASS": DummySpiderLoader + } def syntax(self) -> str: return "[options] " @@ -50,7 +52,7 @@ class Command(BaseRunSpiderCommand): try: module = _import_file(filename) except (ImportError, ValueError) as e: - raise UsageError(f"Unable to load {str(filename)!r}: {e}\n") + raise UsageError(f"Unable to load {str(filename)!r}: {e}\n") from e spclasses = list(iter_spider_classes(module)) if not spclasses: raise UsageError(f"No spider found in file: {filename}\n") diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py index 704cc500d..26a97ccbb 100644 --- a/scrapy/commands/settings.py +++ b/scrapy/commands/settings.py @@ -1,5 +1,6 @@ import argparse import json +from typing import Any, ClassVar from scrapy.commands import ScrapyCommand from scrapy.settings import BaseSettings @@ -7,7 +8,7 @@ from scrapy.settings import BaseSettings class Command(ScrapyCommand): requires_crawler_process = False - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def syntax(self) -> str: return "[options]" diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 772b2911c..8db5f4b49 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -7,7 +7,7 @@ See documentation in docs/topics/shell.rst from __future__ import annotations from threading import Thread -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from scrapy.commands import ScrapyCommand from scrapy.http import Request @@ -23,7 +23,7 @@ if TYPE_CHECKING: class Command(ScrapyCommand): - default_settings = { + default_settings: ClassVar[dict[str, Any]] = { "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", "KEEP_ALIVE": True, "LOGSTATS_INTERVAL": 0, diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py index 8f4427580..c56a7319a 100644 --- a/scrapy/commands/startproject.py +++ b/scrapy/commands/startproject.py @@ -6,7 +6,7 @@ from importlib.util import find_spec from pathlib import Path from shutil import copy2, copystat, ignore_patterns, move from stat import S_IWUSR as OWNER_WRITE_PERMISSION -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, ClassVar import scrapy from scrapy.commands import ScrapyCommand @@ -34,7 +34,7 @@ def _make_writable(path: Path) -> None: class Command(ScrapyCommand): requires_crawler_process = False - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def syntax(self) -> str: return " [project_dir]" @@ -90,7 +90,7 @@ class Command(ScrapyCommand): _make_writable(dst) def run(self, args: list[str], opts: argparse.Namespace) -> None: - if len(args) not in (1, 2): + if len(args) not in {1, 2}: raise UsageError project_name = args[0] diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py index 30b0e9fd7..1d1985d4e 100644 --- a/scrapy/commands/version.py +++ b/scrapy/commands/version.py @@ -1,4 +1,5 @@ import argparse +from typing import Any, ClassVar import scrapy from scrapy.commands import ScrapyCommand @@ -7,7 +8,7 @@ from scrapy.utils.versions import get_versions class Command(ScrapyCommand): requires_crawler_process = False - default_settings = {"LOG_ENABLED": False} + default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False} def syntax(self) -> str: return "[-v]" diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index cbdb36d2f..f84bbe0d7 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -6,7 +6,7 @@ from collections.abc import AsyncGenerator, Iterable from functools import wraps from inspect import getmembers from types import CoroutineType -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, ClassVar, cast from unittest import TestCase, TestResult from scrapy.http import Request, Response @@ -90,7 +90,7 @@ class Contract: class ContractsManager: - contracts: dict[str, type[Contract]] = {} + contracts: ClassVar[dict[str, type[Contract]]] = {} def __init__(self, contracts: Iterable[type[Contract]]): for contract in contracts: @@ -108,8 +108,8 @@ class ContractsManager: def extract_contracts(self, method: Callable) -> list[Contract]: contracts: list[Contract] = [] assert method.__doc__ is not None - for line in method.__doc__.split("\n"): - line = line.strip() + for line_ in method.__doc__.split("\n"): + line = line_.strip() if line.startswith("@"): m = re.match(r"@(\w+)\s*(.*)", line) diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py index 90f054a87..9b42ca36f 100644 --- a/scrapy/contracts/default.py +++ b/scrapy/contracts/default.py @@ -1,7 +1,7 @@ from __future__ import annotations import json -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from itemadapter import ItemAdapter, is_item @@ -68,7 +68,7 @@ class ReturnsContract(Contract): """ name = "returns" - object_type_verifiers: dict[str | None, Callable[[Any], bool]] = { + object_type_verifiers: ClassVar[dict[str | None, Callable[[Any], bool]]] = { "request": lambda x: isinstance(x, Request), "requests": lambda x: isinstance(x, Request), "item": is_item, @@ -78,7 +78,7 @@ class ReturnsContract(Contract): def __init__(self, *args: Any, **kwargs: Any): super().__init__(*args, **kwargs) - if len(self.args) not in [1, 2, 3]: + if len(self.args) not in {1, 2, 3}: raise ValueError( f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}" ) diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py index 84f241d88..d0f32216c 100644 --- a/scrapy/core/downloader/handlers/datauri.py +++ b/scrapy/core/downloader/handlers/datauri.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from w3lib.url import parse_data_uri @@ -17,9 +17,8 @@ class DataURIDownloadHandler(BaseDownloadHandler): uri = parse_data_uri(request.url) respcls = responsetypes.from_mimetype(uri.media_type) - resp_kwargs: dict[str, Any] = {} if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text": charset = uri.media_type_parameters.get("charset") - resp_kwargs["encoding"] = charset + return respcls(url=request.url, body=uri.data, encoding=charset) - return respcls(url=request.url, body=uri.data, **resp_kwargs) + return respcls(url=request.url, body=uri.data) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index a59aa722b..f080cce48 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -7,6 +7,7 @@ from w3lib.url import file_uri_to_path from scrapy.core.downloader.handlers.base import BaseDownloadHandler from scrapy.responsetypes import responsetypes +from scrapy.utils.asyncio import run_in_thread if TYPE_CHECKING: from scrapy import Request @@ -16,6 +17,6 @@ if TYPE_CHECKING: class FileDownloadHandler(BaseDownloadHandler): async def download_request(self, request: Request) -> Response: filepath = file_uri_to_path(request.url) - body = Path(filepath).read_bytes() # noqa: ASYNC240 + body = await run_in_thread(Path(filepath).read_bytes) respcls = responsetypes.from_args(filename=filepath, body=body) return respcls(url=request.url, body=body) diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py index 8327a3b0e..6258067c1 100644 --- a/scrapy/core/downloader/handlers/ftp.py +++ b/scrapy/core/downloader/handlers/ftp.py @@ -33,7 +33,7 @@ from __future__ import annotations import re from io import BytesIO from pathlib import Path -from typing import TYPE_CHECKING, BinaryIO +from typing import TYPE_CHECKING, BinaryIO, ClassVar from urllib.parse import unquote from twisted.internet.protocol import ClientCreator, Protocol @@ -79,7 +79,7 @@ _CODE_RE = re.compile(r"\d+") class FTPDownloadHandler(BaseDownloadHandler): - CODE_MAPPING: dict[str, int] = { + CODE_MAPPING: ClassVar[dict[str, int]] = { "550": 404, "default": 503, } diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9dd403aca..9d102aff8 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -147,7 +147,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): # issue a callback after `_disconnect_timeout` seconds. # # See also https://github.com/scrapy/scrapy/issues/2653 - delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, []) + delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, ()) try: await maybe_deferred_to_future(d) diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index 19f0c16e9..173545f55 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -265,7 +265,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): def _lose_connection_with_error(self, errors: list[BaseException]) -> None: """Helper function to lose the connection with the error sent as a reason""" - self._conn_lost_errors += errors + self._conn_lost_errors.extend(errors) assert self.transport is not None # typing self.transport.loseConnection() @@ -310,7 +310,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): if isinstance(e, FrameTooLargeError): # hyper-h2 does not drop the connection in this scenario, we # need to abort the connection manually. - self._conn_lost_errors += [e] + self._conn_lost_errors.append(e) assert self.transport is not None # typing self.transport.abortConnection() return diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 39c2eaae8..21ce4942e 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -22,6 +22,8 @@ from scrapy.utils._download_handlers import ( from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: + from collections.abc import Sequence + from hpack import HeaderTuple from scrapy.core.http2.protocol import H2ClientProtocol @@ -150,7 +152,7 @@ class Stream: # flow control window "flow_controlled_size": 0, # Headers received after sending the request - "headers": Headers({}), + "headers": Headers(), } def _cancel(_: Any) -> None: @@ -391,7 +393,7 @@ class Stream: def close( self, reason: StreamCloseReason, - errors: list[BaseException] | None = None, + errors: Sequence[BaseException] | None = None, from_protocol: bool = False, ) -> None: """Based on the reason sent we will handle each case.""" @@ -405,7 +407,7 @@ class Stream: # Have default value of errors as an empty list as # some cases can add a list of exceptions - errors = errors or [] + errors = errors or () if not from_protocol: self._protocol.pop_stream(self.stream_id) @@ -470,7 +472,7 @@ class Stream: self._deferred_response.errback(ResponseFailed(errors)) elif reason is StreamCloseReason.INACTIVE: - errors.insert(0, InactiveStreamClosed(self._request)) + errors = (InactiveStreamClosed(self._request), *errors) self._deferred_response.errback(ResponseFailed(errors)) else: diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py index 5a0aa2197..1b7fb652d 100644 --- a/scrapy/core/scheduler.py +++ b/scrapy/core/scheduler.py @@ -461,9 +461,10 @@ class Scheduler(BaseScheduler): except TypeError: # pragma: no cover warn( f"The __init__ method of {global_object_name(self.pqclass)} " - f"does not support a `start_queue_cls` keyword-only " - f"parameter.", + "does not support a `start_queue_cls` keyword-only " + "parameter.", ScrapyDeprecationWarning, + stacklevel=2, ) return build_from_crawler( self.pqclass, @@ -490,9 +491,10 @@ class Scheduler(BaseScheduler): except TypeError: # pragma: no cover warn( f"The __init__ method of {global_object_name(self.pqclass)} " - f"does not support a `start_queue_cls` keyword-only " - f"parameter.", + "does not support a `start_queue_cls` keyword-only " + "parameter.", ScrapyDeprecationWarning, + stacklevel=2, ) q = build_from_crawler( self.pqclass, @@ -521,7 +523,7 @@ class Scheduler(BaseScheduler): def _read_dqs_state(self, dqdir: str) -> Any: path = Path(dqdir, "active.json") if not path.exists(): - return [] + return () with path.open(encoding="utf-8") as f: return json.load(f) diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 007d03a04..39342ebc0 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -114,6 +114,7 @@ class SpiderMiddlewareManager(MiddlewareManager): f"Scrapy 2.13 for details: " f"https://docs.scrapy.org/en/2.13/news.html", ScrapyDeprecationWarning, + stacklevel=2, ) def _add_middleware(self, mw: Any) -> None: @@ -502,6 +503,7 @@ class SpiderMiddlewareManager(MiddlewareManager): f"copy-pasting. See the release notes of Scrapy 2.13 for " f"details: https://docs.scrapy.org/en/2.13/news.html", ScrapyDeprecationWarning, + stacklevel=2, ) if ( diff --git a/scrapy/crawler.py b/scrapy/crawler.py index d622e407c..500139566 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -663,7 +663,7 @@ class CrawlerProcessBase(CrawlerRunnerBase): resolver_class = load_object(self.settings["DNS_RESOLVER"]) # We pass self, which is CrawlerProcess, instead of Crawler here, # which works because the default resolvers only use crawler.settings. - resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type] + resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[call-overload] resolver.install_on_reactor() tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py index d945546d5..cd8c2abca 100644 --- a/scrapy/downloadermiddlewares/cookies.py +++ b/scrapy/downloadermiddlewares/cookies.py @@ -139,7 +139,7 @@ class CookiesMiddleware: for key in ("name", "value", "path", "domain"): value = cookie.get(key) if value is None: - if key in ("name", "value"): + if key in {"name", "value"}: msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)" logger.warning(msg) return None @@ -176,7 +176,7 @@ class CookiesMiddleware: Extract cookies from the Request.cookies attribute """ if not request.cookies: - return [] + return () cookies: Iterable[VerboseCookie] if isinstance(request.cookies, dict): cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items()) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index a176dcd9f..c6c811809 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -107,7 +107,7 @@ class HttpCacheMiddleware: return response # Skip cached responses and uncacheable requests - if "cached" in response.flags or "_dont_cache" in request.meta: + if "_dont_cache" in request.meta or "cached" in response.flags: request.meta.pop("_dont_cache", None) return response diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index d4fa2d4d7..414c3d8a3 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -40,12 +40,13 @@ except ImportError: pass else: try: - brotli.Decompressor.can_accept_more_data + brotli.Decompressor.can_accept_more_data # noqa: B018 except AttributeError: # pragma: no cover warnings.warn( "You have brotli installed. But 'br' encoding support now requires " "brotli's or brotlicffi's version >= 1.2.0. Please upgrade " "brotli/brotlicffi to make Scrapy decode 'br' encoded responses.", + stacklevel=2, ) else: ACCEPTED_ENCODINGS.append(b"br") diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index d3d46a947..e034dced1 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -69,7 +69,7 @@ class HttpProxyMiddleware: _scheme = parsed.scheme if ( # 'no_proxy' is only supported by http schemes - _scheme not in ("http", "https") + _scheme not in {"http", "https"} or (parsed.hostname and not proxy_bypass(parsed.hostname)) ) and _scheme in self.proxies: scheme = _scheme diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 33d7ba609..10f19bacc 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -86,13 +86,13 @@ class OffsiteMiddleware: "allowed_domains accepts only domains, not URLs. " f"Ignoring URL entry {domain} in allowed_domains." ) - warnings.warn(message) + warnings.warn(message, stacklevel=2) elif port_pattern.search(domain): message = ( "allowed_domains accepts only domains without ports. " f"Ignoring entry {domain} in allowed_domains." ) - warnings.warn(message) + warnings.warn(message, stacklevel=2) else: domains.append(re.escape(domain)) regex = rf"^(.*\.)?({'|'.join(domains)})$" diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 69c58ca49..821f41699 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -71,7 +71,7 @@ class BaseRedirectMiddleware: return redirect_cls = global_object_name(self.__class__) referer_cls = global_object_name(RefererMiddleware) - if self.__class__ in (RedirectMiddleware, MetaRefreshMiddleware): + if self.__class__ in {RedirectMiddleware, MetaRefreshMiddleware}: replacement = ( f"replace {redirect_cls} with a subclass that overrides the " f"handle_referer() method" @@ -208,14 +208,19 @@ class RedirectMiddleware(BaseRedirectMiddleware): if ( request.meta.get("dont_redirect", False) or response.status - in getattr(self.crawler.spider, "handle_httpstatus_list", []) - or response.status in request.meta.get("handle_httpstatus_list", []) + in getattr(self.crawler.spider, "handle_httpstatus_list", ()) + or response.status in request.meta.get("handle_httpstatus_list", ()) or request.meta.get("handle_httpstatus_all", False) ): return response - allowed_status = (301, 302, 303, 307, 308) - if "Location" not in response.headers or response.status not in allowed_status: + if "Location" not in response.headers or response.status not in { + 301, + 302, + 303, + 307, + 308, + }: return response assert response.headers["Location"] is not None @@ -235,8 +240,8 @@ class RedirectMiddleware(BaseRedirectMiddleware): if urlparse_cached(redirected).scheme not in {"http", "https"}: return response - if (response.status in (301, 302) and request.method == "POST") or ( - response.status == 303 and request.method not in ("GET", "HEAD") + if (response.status in {301, 302} and request.method == "POST") or ( + response.status == 303 and request.method not in {"GET", "HEAD"} ): redirected = self._redirect_request_using_get( request, response, redirected_url diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index fa928678e..ed4f60785 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -200,7 +200,7 @@ class S3FeedStorage(BlockingFeedStorage): try: import boto3.session # noqa: PLC0415 except ImportError: - raise NotConfigured("missing boto3 library") + raise NotConfigured("missing boto3 library") from None u = urlparse(uri) assert u.hostname self.bucketname: str = u.hostname @@ -250,10 +250,19 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - kwargs: dict[str, Any] = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} - self.s3_client.upload_fileobj( - Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs - ) + if self.acl: + self.s3_client.upload_fileobj( + Bucket=self.bucketname, + Key=self.keyname, + Fileobj=file, + ExtraArgs={"ACL": self.acl}, + ) + else: + self.s3_client.upload_fileobj( + Bucket=self.bucketname, + Key=self.keyname, + Fileobj=file, + ) file.close() @@ -468,9 +477,13 @@ class FeedExporter: # End: Backward compatibility for FEED_URI and FEED_FORMAT settings # 'FEEDS' setting takes precedence over 'FEED_URI' - for uri, feed_options in self.settings.getdict("FEEDS").items(): + for settings_uri, feed_options in self.settings.getdict("FEEDS").items(): # handle pathlib.Path objects - uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri() + uri = ( + str(settings_uri) + if not isinstance(settings_uri, Path) + else settings_uri.absolute().as_uri() + ) self.feeds[uri] = feed_complete_default_values_from_settings( feed_options, self.settings ) @@ -692,9 +705,7 @@ class FeedExporter: uri_params_function: str | UriParamsCallableT | None, slot: FeedSlot | None = None, ) -> dict[str, Any]: - params = {} - for k in dir(spider): - params[k] = getattr(spider, k) + params = {k: getattr(spider, k) for k in dir(spider)} utc_now = datetime.now(tz=timezone.utc) params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-") params["batch_time"] = utc_now.isoformat().replace(":", "-") diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index efda50a82..86b066a38 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -106,10 +106,10 @@ class RFC2616Policy: if b"max-age" in cc or b"Expires" in response.headers: return True # Firefox fallbacks this statuses to one year expiration if none is set - if response.status in (300, 301, 308): + if response.status in {300, 301, 308}: return True # Other statuses without expiration requires at least one validator - if response.status in (200, 203, 401): + if response.status in {200, 203, 401}: return b"Last-Modified" in response.headers or b"ETag" in response.headers # Any other is probably not eligible for caching # Makes no sense to cache responses that does not contain expiration @@ -216,7 +216,7 @@ class RFC2616Policy: return (date - lastmodified) / 10 # This request can be cached indefinitely - if response.status in (300, 301, 308): + if response.status in {300, 301, 308}: return self.MAXAGE # Insufficient information to compute freshness lifetime diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index f045e9b14..01af02ba3 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -39,8 +39,8 @@ class MemoryUsage: try: # stdlib's resource module is only available on unix platforms. self.resource = import_module("resource") - except ImportError: - raise NotConfigured + except ImportError as exc: + raise NotConfigured from exc self.crawler: Crawler = crawler self.warned: bool = False diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index 860b97a55..cd35c8165 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -10,6 +10,7 @@ from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call from scrapy.utils.serialize import ScrapyJSONEncoder if TYPE_CHECKING: + from collections.abc import Sequence from json import JSONEncoder from twisted.internet.task import LoopingCall @@ -31,8 +32,8 @@ class PeriodicLog: self, stats: StatsCollector, interval: float = 60.0, - ext_stats: dict[str, Any] = {}, - ext_delta: dict[str, Any] = {}, + ext_stats: dict[str, Any] | None = None, + ext_delta: dict[str, Any] | None = None, ext_timing_enabled: bool = False, ): self.stats: StatsCollector = stats @@ -41,11 +42,19 @@ class PeriodicLog: self.task: AsyncioLoopingCall | LoopingCall | None = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) - self.ext_stats_include: list[str] = ext_stats.get("include", []) - self.ext_stats_exclude: list[str] = ext_stats.get("exclude", []) + self.ext_stats_include: Sequence[str] = ( + ext_stats.get("include", ()) if ext_stats else () + ) + self.ext_stats_exclude: Sequence[str] = ( + ext_stats.get("exclude", ()) if ext_stats else () + ) self.ext_delta_enabled: bool = bool(ext_delta) - self.ext_delta_include: list[str] = ext_delta.get("include", []) - self.ext_delta_exclude: list[str] = ext_delta.get("exclude", []) + self.ext_delta_include: Sequence[str] = ( + ext_delta.get("include", ()) if ext_delta else () + ) + self.ext_delta_exclude: Sequence[str] = ( + ext_delta.get("exclude", ()) if ext_delta else () + ) self.ext_timing_enabled: bool = ext_timing_enabled @classmethod @@ -143,7 +152,7 @@ class PeriodicLog: return {"stats": stats} def param_allowed( - self, stat_name: str, include: list[str], exclude: list[str] + self, stat_name: str, include: Sequence[str], exclude: Sequence[str] ) -> bool: if not include and not exclude: return True diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 47eb4bf93..f05595806 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -25,6 +25,7 @@ if TYPE_CHECKING: warnings.warn( "The scrapy.extensions.statsmailer module is deprecated and will be " "removed in a future release.", + stacklevel=2, category=ScrapyDeprecationWarning, ) diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py index 13d1c85d0..09286606d 100644 --- a/scrapy/http/cookies.py +++ b/scrapy/http/cookies.py @@ -54,14 +54,14 @@ class CookieJar: if not IPV4_RE.search(req_host): hosts = potential_domain_matches(req_host) if "." not in req_host: - hosts += [req_host + ".local"] + hosts.append(req_host + ".local") else: hosts = [req_host] cookies = [] for host in hosts: if host in self.jar._cookies: # type: ignore[attr-defined] - cookies += self.jar._cookies_for_domain(host, wreq) # type: ignore[attr-defined] + cookies.extend(self.jar._cookies_for_domain(host, wreq)) # type: ignore[attr-defined] attrs = self.jar._cookie_attrs(cookies) # type: ignore[attr-defined] if attrs and not wreq.has_header("Cookie"): diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index d2d13b8df..affa14499 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst from __future__ import annotations from collections.abc import Iterable -from typing import TYPE_CHECKING, Any, TypeAlias, cast +from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from parsel.csstranslator import HTMLTranslator @@ -39,7 +39,7 @@ FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None class FormRequest(Request): __slots__ = () - valid_form_methods = ["GET", "POST"] + valid_form_methods: ClassVar[list[str]] = ["GET", "POST"] def __init__( self, *args: Any, formdata: FormdataType = None, **kwargs: Any @@ -153,7 +153,7 @@ def _get_form( try: form = forms[formnumber] except IndexError: - raise IndexError(f"Form number {formnumber} not found in {response}") + raise IndexError(f"Form number {formnumber} not found in {response}") from None return cast("FormElement", form) @@ -167,7 +167,7 @@ def _get_inputs( try: formdata_keys = dict(formdata or ()).keys() except (ValueError, TypeError): - raise ValueError("formdata should be a dict or iterable of tuples") + raise ValueError("formdata should be a dict or iterable of tuples") from None if not formdata: formdata = [] diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 1776bdca8..13fee06d8 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -36,7 +36,9 @@ class JsonRequest(Request): data_passed: bool = data is not None if body_passed and data_passed: - warnings.warn("Both body and data passed. data will be ignored") + warnings.warn( + "Both body and data passed. data will be ignored", stacklevel=2 + ) elif not body_passed and data_passed: kwargs["body"] = self._dumps(data) if "method" not in kwargs: @@ -68,7 +70,9 @@ class JsonRequest(Request): data_passed: bool = data is not None if body_passed and data_passed: - warnings.warn("Both body and data passed. data will be ignored") + warnings.warn( + "Both body and data passed. data will be ignored", stacklevel=2 + ) elif not body_passed and data_passed: kwargs["body"] = self._dumps(data) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 2cc35fef4..7e23df491 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -86,7 +86,7 @@ class Response(object_ref): raise AttributeError( "Response.cb_kwargs not available, this response " "is not tied to any request" - ) + ) from None @property def meta(self) -> dict[str, Any]: @@ -95,7 +95,7 @@ class Response(object_ref): except AttributeError: raise AttributeError( "Response.meta not available, this response is not tied to any request" - ) + ) from None @property def url(self) -> str: diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 012ead519..077b86a33 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -297,7 +297,7 @@ def _url_from_selector(sel: parsel.Selector) -> str: return strip_html5_whitespace(sel.root) if not hasattr(sel.root, "tag"): raise _InvalidSelector(f"Unsupported selector: {sel}") - if sel.root.tag not in ("a", "link"): + if sel.root.tag not in {"a", "link"}: raise _InvalidSelector( f"Only and elements are supported; got <{sel.root.tag}>" ) diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 96a0f523b..11fd62fb2 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -114,8 +114,8 @@ class LxmlParserLinkExtractor: # pseudo lxml.html.HtmlElement.make_links_absolute(base_url) try: if self.strip: - attr_val = strip_html5_whitespace(attr_val) - attr_val = urljoin(base_url, attr_val) + attr_val = strip_html5_whitespace(attr_val) # noqa: PLW2901 this is intended + attr_val = urljoin(base_url, attr_val) # noqa: PLW2901 except ValueError: continue # skipping bogus links else: @@ -245,7 +245,7 @@ class LxmlLinkExtractor: if self.allow_res else [True] ) - denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] + denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else () return any(allowed) and not any(denied) def _process_links(self, links: list[Link]) -> list[Link]: diff --git a/scrapy/mail.py b/scrapy/mail.py index 718548b03..fbd11ad1d 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -55,6 +55,7 @@ def _to_bytes_or_none(text: str | bytes | None) -> bytes | None: warnings.warn( "The scrapy.mail module is deprecated and will be removed in a future release. " "Please use a dedicated Python mail library instead.", + stacklevel=2, category=ScrapyDeprecationWarning, ) diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 707207337..4d06b6dd3 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -18,7 +18,7 @@ from contextlib import suppress from ftplib import FTP from io import BytesIO from pathlib import Path -from typing import IO, TYPE_CHECKING, Any, NoReturn, Protocol, TypedDict, cast +from typing import IO, TYPE_CHECKING, Any, ClassVar, NoReturn, Protocol, TypedDict, cast from urllib.parse import urlparse from itemadapter import ItemAdapter @@ -161,7 +161,7 @@ class S3FilesStore: AWS_VERIFY = None POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler() - HEADERS = { + HEADERS: ClassVar[dict[str, str]] = { "Cache-Control": "max-age=172800", } @@ -226,7 +226,7 @@ class S3FilesStore: Bucket=self.bucket, Key=key_name, Body=buf, - Metadata={k: str(v) for k, v in (meta or {}).items()}, + Metadata={k: str(v) for k, v in meta.items()} if meta else {}, ACL=self.POLICY, **extra, ) @@ -269,7 +269,9 @@ class S3FilesStore: try: kwarg = mapping[key] except KeyError: - raise TypeError(f'Header "{key}" is not supported by botocore') + raise TypeError( + f'Header "{key}" is not supported by botocore' + ) from None extra[kwarg] = value return extra @@ -339,7 +341,7 @@ class GCSFilesStore: blob_path = self._get_blob_path(path) blob = self.bucket.blob(blob_path) blob.cache_control = self.CACHE_CONTROL - blob.metadata = {k: str(v) for k, v in (meta or {}).items()} + blob.metadata = {k: str(v) for k, v in meta.items()} if meta else {} return deferred_from_coro( run_in_thread( blob.upload_from_string, @@ -435,7 +437,7 @@ class FilesPipeline(MediaPipeline): MEDIA_NAME: str = "file" EXPIRES: int = 90 - STORE_SCHEMES: dict[str, type[FilesStoreProtocol]] = { + STORE_SCHEMES: ClassVar[dict[str, type[FilesStoreProtocol]]] = { "": FSFilesStore, "file": FSFilesStore, "s3": S3FilesStore, @@ -656,7 +658,7 @@ class FilesPipeline(MediaPipeline): exc_info=True, extra={"spider": info.spider}, ) - raise FileException(str(exc)) + raise FileException(str(exc)) from exc return { "url": request.url, diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e92f33cca..83d04e6ca 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -11,7 +11,7 @@ import hashlib import warnings from contextlib import suppress from io import BytesIO -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar from itemadapter import ItemAdapter @@ -49,7 +49,7 @@ class ImagesPipeline(FilesPipeline): MIN_WIDTH: int = 0 MIN_HEIGHT: int = 0 EXPIRES: int = 90 - THUMBS: dict[str, tuple[int, int]] = {} + THUMBS: ClassVar[dict[str, tuple[int, int]]] = {} DEFAULT_IMAGES_URLS_FIELD = "image_urls" DEFAULT_IMAGES_RESULT_FIELD = "images" @@ -76,7 +76,7 @@ class ImagesPipeline(FilesPipeline): except ImportError: raise NotConfigured( "ImagesPipeline requires installing Pillow 8.3.2 or later" - ) + ) from None super().__init__(store_uri, crawler=crawler) @@ -191,7 +191,7 @@ class ImagesPipeline(FilesPipeline): *, response_body: BytesIO, ) -> tuple[Image.Image, BytesIO]: - if image.format in ("PNG", "WEBP") and image.mode == "RGBA": + if image.format in {"PNG", "WEBP"} and image.mode == "RGBA": background = self._Image.new("RGBA", image.size, (255, 255, 255)) background.paste(image, image) image = background.convert("RGB") diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 1fe14c9b0..1043da332 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -230,9 +230,9 @@ class MediaPipeline(ABC): if isinstance(result, Failure): # minimize cached information for failure result.cleanFailure() - result.frames = [] + result.frames.clear() if TWISTED_FAILURE_HAS_STACK: - result.stack = [] # type: ignore[method-assign] + result.stack.clear() # This code fixes a memory leak by avoiding to keep references to # the Request and Response objects on the Media Pipeline cache. # diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py index ed57091ba..ad0b36f6b 100644 --- a/scrapy/pqueues.py +++ b/scrapy/pqueues.py @@ -350,8 +350,9 @@ class DownloaderAwarePriorityQueue: self.crawler: Crawler = crawler self.pqueues: dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue - for slot, startprios in (slot_startprios or {}).items(): - self.pqueues[slot] = self.pqfactory(slot, startprios) + if slot_startprios: + for slot, startprios in slot_startprios.items(): + self.pqueues[slot] = self.pqfactory(slot, startprios) def pqfactory( self, slot: str, startprios: Iterable[int] = () diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py index 3aaf17b53..cd62f02af 100644 --- a/scrapy/responsetypes.py +++ b/scrapy/responsetypes.py @@ -8,7 +8,7 @@ from __future__ import annotations from io import StringIO from mimetypes import MimeTypes from pkgutil import get_data -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ClassVar from scrapy.http import Response from scrapy.utils.misc import load_object @@ -19,7 +19,7 @@ if TYPE_CHECKING: class ResponseTypes: - CLASSES = { + CLASSES: ClassVar[dict[str, str]] = { "text/html": "scrapy.http.HtmlResponse", "application/atom+xml": "scrapy.http.XmlResponse", "application/rdf+xml": "scrapy.http.XmlResponse", diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 598a746a9..d5c4b2dd3 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -184,15 +184,15 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): try: return bool(int(got)) except ValueError: - if got in ("True", "true"): + if got in {"True", "true"}: return True - if got in ("False", "false"): + if got in {"False", "false"}: return False raise ValueError( "Supported values for boolean settings " "are 0/1, True/False, '0'/'1', " "'True'/'False' and 'true'/'false'" - ) + ) from None def getint(self, name: _SettingsKey, default: int = 0) -> int: """ diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index c37b2e759..8c980fd46 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -77,6 +77,7 @@ class SpiderLoader: warnings.warn( "There are several spiders with the same name:\n\n" f"{dupes_string}\n\n This can cause unexpected behavior.", + stacklevel=2, category=UserWarning, ) @@ -96,6 +97,7 @@ class SpiderLoader: f"\n{traceback.format_exc()}Could not load spiders " f"from module '{name}'. " "See above traceback for details.", + stacklevel=2, category=RuntimeWarning, ) else: @@ -114,7 +116,7 @@ class SpiderLoader: try: return self._spiders[spider_name] except KeyError: - raise KeyError(f"Spider not found: {spider_name}") + raise KeyError(f"Spider not found: {spider_name}") from None def find_by_request(self, request: Request) -> list[str]: """ diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index c160d1adb..94b6dfbb5 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -88,5 +88,5 @@ class HttpErrorMiddleware: {"response": response}, extra={"spider": self.crawler.spider}, ) - return [] + return () return None diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index c1c9044a5..6c5acf0de 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -103,7 +103,7 @@ class ReferrerPolicy(ABC): return self.tls_protected(url) def tls_protected(self, url: str) -> bool: - return urlparse(url).scheme in ("https", "ftps") + return urlparse(url).scheme in {"https", "ftps"} class NoReferrerPolicy(ReferrerPolicy): @@ -424,7 +424,7 @@ class RefererMiddleware(BaseSpiderMiddleware): msg += " (import paths from the response Referrer-Policy header are not allowed)" if not warning_only: raise RuntimeError(msg) - warnings.warn(msg, RuntimeWarning) + warnings.warn(msg, RuntimeWarning, stacklevel=2) return None def get_processed_request( diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 89421cbe4..f0d093c6e 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -102,11 +102,12 @@ class CrawlSpider(Spider): self._compile_rules() if method_is_overridden(self.__class__, CrawlSpider, "_parse_response"): warnings.warn( - f"The CrawlSpider._parse_response method, which the " + "The CrawlSpider._parse_response method, which the " f"{global_object_name(self.__class__)} class overrides, is " - f"deprecated: it will be removed in future Scrapy releases. " - f"Please override the CrawlSpider.parse_with_rules method " - f"instead." + "deprecated: it will be removed in future Scrapy releases. " + "Please override the CrawlSpider.parse_with_rules method " + "instead.", + stacklevel=2, ) def _parse(self, response: Response, **kwargs: Any) -> Any: @@ -118,7 +119,7 @@ class CrawlSpider(Spider): ) def parse_start_url(self, response: Response, **kwargs: Any) -> Any: - return [] + return () def process_results( self, response: Response, results: Iterable[Any] @@ -209,8 +210,9 @@ class CrawlSpider(Spider): def _compile_rules(self) -> None: self._rules = [] for rule in self.rules: - self._rules.append(copy.copy(rule)) - self._rules[-1]._compile(self) + copied_rule = copy.copy(rule) + copied_rule._compile(self) + self._rules.append(copied_rule) @classmethod def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 2813a32a0..cb5779d74 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -49,7 +49,7 @@ class SitemapSpider(Spider): self._cbs: list[tuple[re.Pattern[str], CallbackT]] = [] for r, c in self.sitemap_rules: if isinstance(c, str): - c = cast("CallbackT", getattr(self, c)) + c = cast("CallbackT", getattr(self, c)) # noqa: PLW2901 self._cbs.append((regex(r), c)) self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index a3df08cd9..93fbb9033 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -13,7 +13,7 @@ KnownShellsT = dict[str, Callable[..., EmbedFuncT]] def _embed_ipython_shell( - namespace: dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] | None = None, banner: str = "" ) -> EmbedFuncT: """Start an IPython Shell""" try: @@ -28,7 +28,7 @@ def _embed_ipython_shell( ) @wraps(_embed_ipython_shell) - def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: config = load_default_config() # type: ignore[no-untyped-call] # Always use .instance() to ensure _instance propagation to all parents # this is needed for completion works well for new imports @@ -44,26 +44,26 @@ def _embed_ipython_shell( def _embed_bpython_shell( - namespace: dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] | None = None, banner: str = "" ) -> EmbedFuncT: """Start a bpython shell""" import bpython # noqa: PLC0415 @wraps(_embed_bpython_shell) - def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: bpython.embed(locals_=namespace, banner=banner) return wrapper def _embed_ptpython_shell( - namespace: dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] | None = None, banner: str = "" ) -> EmbedFuncT: """Start a ptpython shell""" import ptpython.repl # noqa: PLC0415 # pylint: disable=import-error @wraps(_embed_ptpython_shell) - def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: print(banner) ptpython.repl.embed(locals=namespace) @@ -71,7 +71,7 @@ def _embed_ptpython_shell( def _embed_standard_shell( - namespace: dict[str, Any] = {}, banner: str = "" + namespace: dict[str, Any] | None = None, banner: str = "" ) -> EmbedFuncT: """Start a standard python shell""" try: # readline module is only available on unix systems @@ -84,7 +84,7 @@ def _embed_standard_shell( readline.parse_and_bind("tab:complete") # type: ignore[attr-defined,unused-ignore] @wraps(_embed_standard_shell) - def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None: + def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None: code.interact(banner=banner, local=namespace) return wrapper diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index a40ee8997..646335fb1 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -106,7 +106,7 @@ def curl_to_request_kwargs( if argv: msg = f"Unrecognized options: {', '.join(argv)}" if ignore_unknown_options: - warnings.warn(msg) + warnings.warn(msg, stacklevel=2) else: raise ValueError(msg) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 1acb576b4..7d7636fe6 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -8,6 +8,7 @@ import asyncio import inspect import warnings from asyncio import Future +from collections import deque from collections.abc import Awaitable, Coroutine, Iterable, Iterator from functools import wraps from typing import ( @@ -230,7 +231,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): self.callable_args: tuple[Any, ...] = callable_args self.callable_kwargs: dict[str, Any] = callable_kwargs self.finished: bool = False - self.waiting_deferreds: list[Deferred[Any]] = [] + self.waiting_deferreds: deque[Deferred[Any]] = deque() self.anext_deferred: Deferred[_T] | None = None def _callback(self, result: _T) -> None: @@ -241,7 +242,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): callable_result = self.callable( result, *self.callable_args, **self.callable_kwargs ) - d = self.waiting_deferreds.pop(0) + d = self.waiting_deferreds.popleft() if isinstance(callable_result, Deferred): callable_result.chainDeferred(d) else: diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py index 3bf6639c5..da1838030 100644 --- a/scrapy/utils/deprecate.py +++ b/scrapy/utils/deprecate.py @@ -130,7 +130,7 @@ def create_deprecated_class( # deprecated class is in jinja2 template). __module__ attribute is not # important enough to raise an exception as users may be unable # to fix inspect.stack() errors. - warnings.warn(f"Error detecting parent module: {e!r}") + warnings.warn(f"Error detecting parent module: {e!r}", stacklevel=2) return deprecated_cls @@ -160,6 +160,7 @@ def update_classpath(path: Any) -> Any: warnings.warn( f"`{path}` class is deprecated, use `{new_path}` instead", ScrapyDeprecationWarning, + stacklevel=2, ) return new_path return path diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index ee988a665..fba80484c 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -13,7 +13,7 @@ from contextlib import contextmanager from functools import partial from importlib import import_module from pkgutil import iter_modules -from typing import IO, TYPE_CHECKING, Any, TypeVar, cast +from typing import IO, TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, overload from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.item import Item @@ -28,9 +28,20 @@ if TYPE_CHECKING: _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes -T = TypeVar("T") +_ITER_T = TypeVar("_ITER_T", bound=dict | Item | str | bytes) +_T = TypeVar("_T") +_T_co = TypeVar("_T_co", covariant=True) +_P = ParamSpec("_P") +@overload +def arg_to_iter(arg: None) -> tuple[()]: ... +@overload +def arg_to_iter(arg: _ITER_T) -> Iterable[_ITER_T]: ... +@overload +def arg_to_iter(arg: Iterable[_T]) -> Iterable[_T]: ... +@overload +def arg_to_iter(arg: _T) -> Iterable[_T]: ... def arg_to_iter(arg: Any) -> Iterable[Any]: """Convert an argument to an iterable. The argument can be a None, single value, or an iterable. @@ -38,9 +49,9 @@ def arg_to_iter(arg: Any) -> Iterable[Any]: Exception: if arg is a dict, [arg] will be returned """ if arg is None: - return [] + return () if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"): - return cast("Iterable[Any]", arg) + return arg return [arg] @@ -64,7 +75,7 @@ def load_object(path: str | Callable[..., Any]) -> Any: try: dot = path.rindex(".") except ValueError: - raise ValueError(f"Error loading object '{path}': not a full path") + raise ValueError(f"Error loading object '{path}': not a full path") from None module, name = path[:dot], path[dot + 1 :] mod = import_module(module) @@ -72,7 +83,9 @@ def load_object(path: str | Callable[..., Any]) -> Any: try: obj = getattr(mod, name) except AttributeError: - raise NameError(f"Module '{module}' doesn't define any object named '{name}'") + raise NameError( + f"Module '{module}' doesn't define any object named '{name}'" + ) from None return obj @@ -152,9 +165,40 @@ def rel_has_nofollow(rel: str | None) -> bool: return rel is not None and "nofollow" in rel.replace(",", " ").split() +class SupportsFromCrawler(Protocol[_T_co, _P]): + @classmethod + def from_crawler( + cls, crawler: Crawler, /, *args: _P.args, **kwargs: _P.kwargs + ) -> _T_co: ... + + +@overload def build_from_crawler( - objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any -) -> T: + objcls: SupportsFromCrawler[_T_co, _P], + crawler: Crawler, + /, + *args: _P.args, + **kwargs: _P.kwargs, +) -> _T_co: ... + + +@overload +def build_from_crawler( + objcls: Callable[_P, _T_co], + crawler: Crawler, + /, + *args: _P.args, + **kwargs: _P.kwargs, +) -> _T_co: ... + + +def build_from_crawler( + objcls: Any, + crawler: Crawler, + /, + *args: Any, + **kwargs: Any, +) -> Any: """Construct a class instance using its ``from_crawler()`` or ``__init__()`` constructor. .. versionadded:: 2.12 @@ -164,14 +208,14 @@ def build_from_crawler( Raises ``TypeError`` if the resulting instance is ``None``. """ if hasattr(objcls, "from_crawler"): - instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined] + instance = objcls.from_crawler(crawler, *args, **kwargs) method_name = "from_crawler" else: instance = objcls(*args, **kwargs) method_name = "__new__" if instance is None: raise TypeError(f"{objcls.__qualname__}.{method_name} returned None") - return cast("T", instance) + return instance @contextmanager diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 0139720b7..3e75c7729 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -20,7 +20,8 @@ def inside_project() -> bool: import_module(scrapy_module) except ImportError as exc: warnings.warn( - f"Cannot import scrapy settings module {scrapy_module}: {exc}" + f"Cannot import scrapy settings module {scrapy_module}: {exc}", + stacklevel=2, ) else: return True diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 419c552bf..b41b8af38 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -76,7 +76,7 @@ class CallLaterOnce(Generic[_T]): for d in self._deferreds: call_later(0, d.callback, None) - self._deferreds = [] + self._deferreds.clear() return result diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 616ab661f..27d669e71 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -169,7 +169,7 @@ def _get_method(obj: Any, name: Any) -> Any: try: return getattr(obj, name) except AttributeError: - raise ValueError(f"Method {name!r} not found in: {obj}") + raise ValueError(f"Method {name!r} not found in: {obj}") from None def request_to_curl(request: Request) -> str: diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 2f1acffd7..7d548524a 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -4,7 +4,7 @@ from __future__ import annotations import os import sys import warnings -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, ClassVar, cast from twisted.internet.defer import Deferred from twisted.internet.protocol import ProcessProtocol @@ -21,12 +21,13 @@ if TYPE_CHECKING: warnings.warn( "The scrapy.utils.testproc module is deprecated.", ScrapyDeprecationWarning, + stacklevel=2, ) class ProcessTest: command: str | None = None - prefix = [sys.executable, "-m", "scrapy.cmdline"] + prefix: ClassVar[list[str]] = [sys.executable, "-m", "scrapy.cmdline"] cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109 def execute( diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index e57eb802b..4089e86ed 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -9,6 +9,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning warnings.warn( "The scrapy.utils.testsite module is deprecated.", ScrapyDeprecationWarning, + stacklevel=2, ) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 097809cac..39b581ff0 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -19,13 +19,18 @@ from w3lib.url import parse_url as _parse_url from scrapy.exceptions import ScrapyDeprecationWarning +_DEPRECATED_NAMES: frozenset[str] = frozenset( + {"_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects} +) + def __getattr__(name: str) -> Any: - if name in ("_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects): + if name in _DEPRECATED_NAMES: obj_type = "attribute" if name == "_safe_chars" else "function" warnings.warn( f"The scrapy.utils.url.{name} {obj_type} is deprecated, use w3lib.url.{name} instead.", ScrapyDeprecationWarning, + stacklevel=2, ) return getattr(import_module("w3lib.url"), name) @@ -45,15 +50,18 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool: host = _parse_url(url).netloc.lower() if not host: return False - domains = [d.lower() for d in domains] - return any((host == d) or (host.endswith(f".{d}")) for d in domains) + return any((host == d) or (host.endswith(f".{d}")) for d in map(str.lower, domains)) + + +def _spider_domains(spider: type[Spider]) -> Iterable[str]: + yield spider.name + if allowed_domains := getattr(spider, "allowed_domains", None): + yield from allowed_domains def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool: """Return True if the url belongs to the given spider""" - return url_is_from_any_domain( - url, [spider.name, *getattr(spider, "allowed_domains", [])] - ) + return url_is_from_any_domain(url, _spider_domains(spider)) def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool: @@ -184,11 +192,11 @@ def strip_url( strip_default_port and parsed_url.port and (parsed_url.scheme, parsed_url.port) - in ( + in { ("http", 80), ("https", 443), ("ftp", 21), - ) + } ): netloc = netloc.replace(f":{parsed_url.port}", "") diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index b15f68dd0..d5ad59346 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -88,7 +88,7 @@ class TestHttpErrorMiddleware: def test_process_spider_exception( self, mw: HttpErrorMiddleware, res404: Response ) -> None: - assert mw.process_spider_exception(res404, HttpError(res404)) == [] + assert mw.process_spider_exception(res404, HttpError(res404)) == () assert mw.process_spider_exception(res404, Exception()) is None def test_handle_httpstatus_list(