RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chore: more ruff rules and overall minor improvements (#7386) 

* overall prettyfication

- don't create empty mutable containers (lists, dicts) where it is appropriate
- removed from ignore section and applied some rules from ruff (but keep them ignored in tests)
- use `deque` in `_AsyncCooperatorAdapter` instead of `list.pop(0)`
- remove `f` prefix from strings without any formatting

* return `SIM300` to ignore

* apply ruff rules to all files

* another one

* remove extra space in pyproject.toml

* simplify `__getattr__` in `scrapy.utils.url`

* lazy `url_is_from_any_domain` and `url_is_from_spider`

* improve typing

* specify `arg_to_iter` with None as arg
This commit is contained in:
Albert Eduardovich N. 2026-04-03 17:00:13 +03:00 committed by GitHub
parent e3a8ff2b59
commit 58d85282cf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
66 changed files with 308 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
pyproject.toml
View File

@ -415,25 +415,6 @@ ignore = [
"SIM115",
# Yoda condition detected
"SIM300",
# Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
# Assigning to `os.environ` doesn't clear the environment.
"B003",
# Do not use mutable data structures for argument defaults.
"B006",
# Found useless expression.
"B018",
# No explicit stacklevel argument found.
"B028",
# Within an `except` clause, raise exceptions with `raise ... from`
"B904",
# `for` loop variable overwritten by assignment target
"PLW2901",
# Mutable class attributes should be annotated with `typing.ClassVar`
"RUF012",
# Use capitalized environment variable
"SIM112",
]
[tool.ruff.lint.flake8-tidy-imports]
@ -453,8 +434,28 @@ split-on-trailing-comma = false
"scrapy/linkextractors/__init__.py" = ["E402"]
"scrapy/spiders/__init__.py" = ["E402"]
# Skip bandit and allow blocking file I/O in tests
"tests/**" = ["ASYNC240", "S"]
"tests/**" = [
# Skip bandit and allow blocking file I/O in tests
"ASYNC240",
"S",
# Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
# Assigning to `os.environ` doesn't clear the environment.
"B003",
# Do not use mutable data structures for argument defaults.
"B006",
# Found useless expression.
"B018",
# No explicit stacklevel argument found.
"B028",
# Within an `except` clause, raise exceptions with `raise ... from`
"B904",
# `for` loop variable overwritten by assignment target
"PLW2901",
# Mutable class attributes should be annotated with `typing.ClassVar`
"RUF012",
# Use capitalized environment variable
"SIM112",
]
# Issues pending a review:
"docs/conf.py" = ["E402"]

2
scrapy/cmdline.py
View File

@ -46,7 +46,7 @@ def _iter_command_classes(module_name: str) -> Iterable[type[ScrapyCommand]]:
inspect.isclass(obj)
and issubclass(obj, ScrapyCommand)
and obj.__module__ == module.__name__
and obj not in (ScrapyCommand, BaseRunSpiderCommand)
and obj not in {ScrapyCommand, BaseRunSpiderCommand}
):
yield obj

12
scrapy/commands/__init__.py
View File

@ -10,7 +10,7 @@ import os
import warnings
from abc import ABC, abstractmethod
from pathlib import Path
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, ClassVar
from twisted.python import failure
@ -30,7 +30,7 @@ class ScrapyCommand(ABC):
crawler_process: CrawlerProcessBase | None = None # set in scrapy.cmdline
# default settings to be used for this command instead of global defaults
default_settings: dict[str, Any] = {}
default_settings: ClassVar[dict[str, Any]] = {}
exitcode: int = 0
@ -115,7 +115,9 @@ class ScrapyCommand(ABC):
try:
self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline")
except ValueError:
raise UsageError("Invalid -s value, use -s NAME=VALUE", print_help=False)
raise UsageError(
"Invalid -s value, use -s NAME=VALUE", print_help=False
) from None
if opts.logfile:
self.settings.set("LOG_ENABLED", True, priority="cmdline")
@ -181,7 +183,9 @@ class BaseRunSpiderCommand(ScrapyCommand):
try:
opts.spargs = arglist_to_dict(opts.spargs)
except ValueError:
raise UsageError("Invalid -a value, use -a NAME=VALUE", print_help=False)
raise UsageError(
"Invalid -a value, use -a NAME=VALUE", print_help=False
) from None
if opts.output or opts.overwrite_output:
assert self.settings is not None
feeds = feed_process_params_from_cli(

4
scrapy/commands/bench.py
View File

@ -3,7 +3,7 @@ from __future__ import annotations
import subprocess
import sys
import time
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, ClassVar
from urllib.parse import urlencode
import scrapy
@ -18,7 +18,7 @@ if TYPE_CHECKING:
class Command(ScrapyCommand):
default_settings = {
default_settings: ClassVar[dict[str, Any]] = {
"LOG_LEVEL": "INFO",
"LOGSTATS_INTERVAL": 1,
"CLOSESPIDER_TIMEOUT": 10,

4
scrapy/commands/check.py
View File

@ -2,7 +2,7 @@ import argparse
import time
from collections import defaultdict
from collections.abc import AsyncIterator
from typing import Any
from typing import Any, ClassVar
from unittest import TextTestResult as _TextTestResult
from unittest import TextTestRunner
@ -44,7 +44,7 @@ class TextTestResult(_TextTestResult):
class Command(ScrapyCommand):
requires_project = True
default_settings = {"LOG_ENABLED": False}
default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
def syntax(self) -> str:
return "[options] <spider>"

3
scrapy/commands/edit.py
View File

@ -1,6 +1,7 @@
import argparse
import os
import sys
from typing import Any, ClassVar
from scrapy.commands import ScrapyCommand
from scrapy.exceptions import UsageError
@ -10,7 +11,7 @@ from scrapy.spiderloader import get_spider_loader
class Command(ScrapyCommand):
requires_project = True
requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
def syntax(self) -> str:
return "<spider>"

4
scrapy/commands/genspider.py
View File

@ -5,7 +5,7 @@ import shutil
import string
from importlib import import_module
from pathlib import Path
from typing import TYPE_CHECKING, Any, cast
from typing import TYPE_CHECKING, Any, ClassVar, cast
from urllib.parse import urlparse
import scrapy
@ -47,7 +47,7 @@ def verify_url_scheme(url: str) -> str:
class Command(ScrapyCommand):
requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
def syntax(self) -> str:
return "[options] <name> <domain>"

4
scrapy/commands/list.py
View File

@ -1,6 +1,6 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any, ClassVar
from scrapy.commands import ScrapyCommand
from scrapy.spiderloader import get_spider_loader
@ -12,7 +12,7 @@ if TYPE_CHECKING:
class Command(ScrapyCommand):
requires_project = True
requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
def short_desc(self) -> str:
return "List available spiders"

10
scrapy/commands/parse.py
View File

@ -4,7 +4,7 @@ import functools
import inspect
import json
import logging
from typing import TYPE_CHECKING, Any, TypeVar, overload
from typing import TYPE_CHECKING, Any, ClassVar, TypeVar, overload
from itemadapter import ItemAdapter
from twisted.internet.defer import Deferred, maybeDeferred
@ -39,8 +39,8 @@ class Command(BaseRunSpiderCommand):
requires_project = True
spider: Spider | None = None
items: dict[int, list[Any]] = {}
requests: dict[int, list[Request]] = {}
items: ClassVar[dict[int, list[Any]]] = {}
requests: ClassVar[dict[int, list[Request]]] = {}
spidercls: type[Spider] | None
first_response = None
@ -387,7 +387,7 @@ class Command(BaseRunSpiderCommand):
"Invalid -m/--meta value, pass a valid json string to -m or --meta. "
'Example: --meta=\'{"foo" : "bar"}\'',
print_help=False,
)
) from None
def process_request_cb_kwargs(self, opts: argparse.Namespace) -> None:
if opts.cbkwargs:
@ -398,7 +398,7 @@ class Command(BaseRunSpiderCommand):
"Invalid --cbkwargs value, pass a valid json string to --cbkwargs. "
'Example: --cbkwargs=\'{"foo" : "bar"}\'',
print_help=False,
)
) from None
def run(self, args: list[str], opts: argparse.Namespace) -> None:
# parse arguments

10
scrapy/commands/runspider.py
View File

@ -3,7 +3,7 @@ from __future__ import annotations
import sys
from importlib import import_module
from pathlib import Path
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any, ClassVar
from scrapy.commands import BaseRunSpiderCommand
from scrapy.exceptions import UsageError
@ -18,7 +18,7 @@ if TYPE_CHECKING:
def _import_file(filepath: str | PathLike[str]) -> ModuleType:
abspath = Path(filepath).resolve()
if abspath.suffix not in (".py", ".pyw"):
if abspath.suffix not in {".py", ".pyw"}:
raise ValueError(f"Not a Python source file: {abspath}")
dirname = str(abspath.parent)
sys.path = [dirname, *sys.path]
@ -30,7 +30,9 @@ def _import_file(filepath: str | PathLike[str]) -> ModuleType:
class Command(BaseRunSpiderCommand):
default_settings = {"SPIDER_LOADER_CLASS": DummySpiderLoader}
default_settings: ClassVar[dict[str, Any]] = {
"SPIDER_LOADER_CLASS": DummySpiderLoader
}
def syntax(self) -> str:
return "[options] <spider_file>"
@ -50,7 +52,7 @@ class Command(BaseRunSpiderCommand):
try:
module = _import_file(filename)
except (ImportError, ValueError) as e:
raise UsageError(f"Unable to load {str(filename)!r}: {e}\n")
raise UsageError(f"Unable to load {str(filename)!r}: {e}\n") from e
spclasses = list(iter_spider_classes(module))
if not spclasses:
raise UsageError(f"No spider found in file: {filename}\n")

3
scrapy/commands/settings.py
View File

@ -1,5 +1,6 @@
import argparse
import json
from typing import Any, ClassVar
from scrapy.commands import ScrapyCommand
from scrapy.settings import BaseSettings
@ -7,7 +8,7 @@ from scrapy.settings import BaseSettings
class Command(ScrapyCommand):
requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
def syntax(self) -> str:
return "[options]"

4
scrapy/commands/shell.py
View File

@ -7,7 +7,7 @@ See documentation in docs/topics/shell.rst
from __future__ import annotations
from threading import Thread
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, ClassVar
from scrapy.commands import ScrapyCommand
from scrapy.http import Request
@ -23,7 +23,7 @@ if TYPE_CHECKING:
class Command(ScrapyCommand):
default_settings = {
default_settings: ClassVar[dict[str, Any]] = {
"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
"KEEP_ALIVE": True,
"LOGSTATS_INTERVAL": 0,

6
scrapy/commands/startproject.py
View File

@ -6,7 +6,7 @@ from importlib.util import find_spec
from pathlib import Path
from shutil import copy2, copystat, ignore_patterns, move
from stat import S_IWUSR as OWNER_WRITE_PERMISSION
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any, ClassVar
import scrapy
from scrapy.commands import ScrapyCommand
@ -34,7 +34,7 @@ def _make_writable(path: Path) -> None:
class Command(ScrapyCommand):
requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
def syntax(self) -> str:
return "<project_name> [project_dir]"
@ -90,7 +90,7 @@ class Command(ScrapyCommand):
_make_writable(dst)
def run(self, args: list[str], opts: argparse.Namespace) -> None:
if len(args) not in (1, 2):
if len(args) not in {1, 2}:
raise UsageError
project_name = args[0]

3
scrapy/commands/version.py
View File

@ -1,4 +1,5 @@
import argparse
from typing import Any, ClassVar
import scrapy
from scrapy.commands import ScrapyCommand
@ -7,7 +8,7 @@ from scrapy.utils.versions import get_versions
class Command(ScrapyCommand):
requires_crawler_process = False
default_settings = {"LOG_ENABLED": False}
default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
def syntax(self) -> str:
return "[-v]"

8
scrapy/contracts/__init__.py
View File

@ -6,7 +6,7 @@ from collections.abc import AsyncGenerator, Iterable
from functools import wraps
from inspect import getmembers
from types import CoroutineType
from typing import TYPE_CHECKING, Any, cast
from typing import TYPE_CHECKING, Any, ClassVar, cast
from unittest import TestCase, TestResult
from scrapy.http import Request, Response
@ -90,7 +90,7 @@ class Contract:
class ContractsManager:
contracts: dict[str, type[Contract]] = {}
contracts: ClassVar[dict[str, type[Contract]]] = {}
def __init__(self, contracts: Iterable[type[Contract]]):
for contract in contracts:
@ -108,8 +108,8 @@ class ContractsManager:
def extract_contracts(self, method: Callable) -> list[Contract]:
contracts: list[Contract] = []
assert method.__doc__ is not None
for line in method.__doc__.split("\n"):
line = line.strip()
for line_ in method.__doc__.split("\n"):
line = line_.strip()
if line.startswith("@"):
m = re.match(r"@(\w+)\s*(.*)", line)

6
scrapy/contracts/default.py
View File

@ -1,7 +1,7 @@
from __future__ import annotations
import json
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, ClassVar
from itemadapter import ItemAdapter, is_item
@ -68,7 +68,7 @@ class ReturnsContract(Contract):
"""
name = "returns"
object_type_verifiers: dict[str | None, Callable[[Any], bool]] = {
object_type_verifiers: ClassVar[dict[str | None, Callable[[Any], bool]]] = {
"request": lambda x: isinstance(x, Request),
"requests": lambda x: isinstance(x, Request),
"item": is_item,
@ -78,7 +78,7 @@ class ReturnsContract(Contract):
def __init__(self, *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
if len(self.args) not in [1, 2, 3]:
if len(self.args) not in {1, 2, 3}:
raise ValueError(
f"Incorrect argument quantity: expected 1, 2 or 3, got {len(self.args)}"
)

7
scrapy/core/downloader/handlers/datauri.py
View File

@ -1,6 +1,6 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING
from w3lib.url import parse_data_uri
@ -17,9 +17,8 @@ class DataURIDownloadHandler(BaseDownloadHandler):
uri = parse_data_uri(request.url)
respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs: dict[str, Any] = {}
if issubclass(respcls, TextResponse) and uri.media_type.split("/")[0] == "text":
charset = uri.media_type_parameters.get("charset")
resp_kwargs["encoding"] = charset
return respcls(url=request.url, body=uri.data, encoding=charset)
return respcls(url=request.url, body=uri.data, **resp_kwargs)
return respcls(url=request.url, body=uri.data)

3
scrapy/core/downloader/handlers/file.py
View File

@ -7,6 +7,7 @@ from w3lib.url import file_uri_to_path
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.responsetypes import responsetypes
from scrapy.utils.asyncio import run_in_thread
if TYPE_CHECKING:
from scrapy import Request
@ -16,6 +17,6 @@ if TYPE_CHECKING:
class FileDownloadHandler(BaseDownloadHandler):
async def download_request(self, request: Request) -> Response:
filepath = file_uri_to_path(request.url)
body = Path(filepath).read_bytes() # noqa: ASYNC240
body = await run_in_thread(Path(filepath).read_bytes)
respcls = responsetypes.from_args(filename=filepath, body=body)
return respcls(url=request.url, body=body)

4
scrapy/core/downloader/handlers/ftp.py
View File

@ -33,7 +33,7 @@ from __future__ import annotations
import re
from io import BytesIO
from pathlib import Path
from typing import TYPE_CHECKING, BinaryIO
from typing import TYPE_CHECKING, BinaryIO, ClassVar
from urllib.parse import unquote
from twisted.internet.protocol import ClientCreator, Protocol
@ -79,7 +79,7 @@ _CODE_RE = re.compile(r"\d+")
class FTPDownloadHandler(BaseDownloadHandler):
CODE_MAPPING: dict[str, int] = {
CODE_MAPPING: ClassVar[dict[str, int]] = {
"550": 404,
"default": 503,
}

2
scrapy/core/downloader/handlers/http11.py
View File

@ -147,7 +147,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
# issue a callback after `_disconnect_timeout` seconds.
#
# See also https://github.com/scrapy/scrapy/issues/2653
delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, [])
delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, ())
try:
await maybe_deferred_to_future(d)

4
scrapy/core/http2/protocol.py
View File

@ -265,7 +265,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
def _lose_connection_with_error(self, errors: list[BaseException]) -> None:
"""Helper function to lose the connection with the error sent as a
reason"""
self._conn_lost_errors += errors
self._conn_lost_errors.extend(errors)
assert self.transport is not None # typing
self.transport.loseConnection()
@ -310,7 +310,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
if isinstance(e, FrameTooLargeError):
# hyper-h2 does not drop the connection in this scenario, we
# need to abort the connection manually.
self._conn_lost_errors += [e]
self._conn_lost_errors.append(e)
assert self.transport is not None # typing
self.transport.abortConnection()
return

10
scrapy/core/http2/stream.py
View File

@ -22,6 +22,8 @@ from scrapy.utils._download_handlers import (
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
from collections.abc import Sequence
from hpack import HeaderTuple
from scrapy.core.http2.protocol import H2ClientProtocol
@ -150,7 +152,7 @@ class Stream:
# flow control window
"flow_controlled_size": 0,
# Headers received after sending the request
"headers": Headers({}),
"headers": Headers(),
}
def _cancel(_: Any) -> None:
@ -391,7 +393,7 @@ class Stream:
def close(
self,
reason: StreamCloseReason,
errors: list[BaseException] | None = None,
errors: Sequence[BaseException] | None = None,
from_protocol: bool = False,
) -> None:
"""Based on the reason sent we will handle each case."""
@ -405,7 +407,7 @@ class Stream:
# Have default value of errors as an empty list as
# some cases can add a list of exceptions
errors = errors or []
errors = errors or ()
if not from_protocol:
self._protocol.pop_stream(self.stream_id)
@ -470,7 +472,7 @@ class Stream:
self._deferred_response.errback(ResponseFailed(errors))
elif reason is StreamCloseReason.INACTIVE:
errors.insert(0, InactiveStreamClosed(self._request))
errors = (InactiveStreamClosed(self._request), *errors)
self._deferred_response.errback(ResponseFailed(errors))
else:

12
scrapy/core/scheduler.py
View File

@ -461,9 +461,10 @@ class Scheduler(BaseScheduler):
except TypeError: # pragma: no cover
warn(
f"The __init__ method of {global_object_name(self.pqclass)} "
f"does not support a `start_queue_cls` keyword-only "
f"parameter.",
"does not support a `start_queue_cls` keyword-only "
"parameter.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return build_from_crawler(
self.pqclass,
@ -490,9 +491,10 @@ class Scheduler(BaseScheduler):
except TypeError: # pragma: no cover
warn(
f"The __init__ method of {global_object_name(self.pqclass)} "
f"does not support a `start_queue_cls` keyword-only "
f"parameter.",
"does not support a `start_queue_cls` keyword-only "
"parameter.",
ScrapyDeprecationWarning,
stacklevel=2,
)
q = build_from_crawler(
self.pqclass,
@ -521,7 +523,7 @@ class Scheduler(BaseScheduler):
def _read_dqs_state(self, dqdir: str) -> Any:
path = Path(dqdir, "active.json")
if not path.exists():
return []
return ()
with path.open(encoding="utf-8") as f:
return json.load(f)

2
scrapy/core/spidermw.py
View File

@ -114,6 +114,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
f"Scrapy 2.13 for details: "
f"https://docs.scrapy.org/en/2.13/news.html",
ScrapyDeprecationWarning,
stacklevel=2,
)
def _add_middleware(self, mw: Any) -> None:
@ -502,6 +503,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
f"copy-pasting. See the release notes of Scrapy 2.13 for "
f"details: https://docs.scrapy.org/en/2.13/news.html",
ScrapyDeprecationWarning,
stacklevel=2,
)
if (

2
scrapy/crawler.py
View File

@ -663,7 +663,7 @@ class CrawlerProcessBase(CrawlerRunnerBase):
resolver_class = load_object(self.settings["DNS_RESOLVER"])
# We pass self, which is CrawlerProcess, instead of Crawler here,
# which works because the default resolvers only use crawler.settings.
resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type]
resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[call-overload]
resolver.install_on_reactor()
tp = reactor.getThreadPool()
tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))

4
scrapy/downloadermiddlewares/cookies.py
View File

@ -139,7 +139,7 @@ class CookiesMiddleware:
for key in ("name", "value", "path", "domain"):
value = cookie.get(key)
if value is None:
if key in ("name", "value"):
if key in {"name", "value"}:
msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
logger.warning(msg)
return None
@ -176,7 +176,7 @@ class CookiesMiddleware:
Extract cookies from the Request.cookies attribute
"""
if not request.cookies:
return []
return ()
cookies: Iterable[VerboseCookie]
if isinstance(request.cookies, dict):
cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items())

2
scrapy/downloadermiddlewares/httpcache.py
View File

@ -107,7 +107,7 @@ class HttpCacheMiddleware:
return response
# Skip cached responses and uncacheable requests
if "cached" in response.flags or "_dont_cache" in request.meta:
if "_dont_cache" in request.meta or "cached" in response.flags:
request.meta.pop("_dont_cache", None)
return response

3
scrapy/downloadermiddlewares/httpcompression.py
View File

@ -40,12 +40,13 @@ except ImportError:
pass
else:
try:
brotli.Decompressor.can_accept_more_data
brotli.Decompressor.can_accept_more_data # noqa: B018
except AttributeError: # pragma: no cover
warnings.warn(
"You have brotli installed. But 'br' encoding support now requires "
"brotli's or brotlicffi's version >= 1.2.0. Please upgrade "
"brotli/brotlicffi to make Scrapy decode 'br' encoded responses.",
stacklevel=2,
)
else:
ACCEPTED_ENCODINGS.append(b"br")

2
scrapy/downloadermiddlewares/httpproxy.py
View File

@ -69,7 +69,7 @@ class HttpProxyMiddleware:
_scheme = parsed.scheme
if (
# 'no_proxy' is only supported by http schemes
_scheme not in ("http", "https")
_scheme not in {"http", "https"}
or (parsed.hostname and not proxy_bypass(parsed.hostname))
) and _scheme in self.proxies:
scheme = _scheme

4
scrapy/downloadermiddlewares/offsite.py
View File

@ -86,13 +86,13 @@ class OffsiteMiddleware:
"allowed_domains accepts only domains, not URLs. "
f"Ignoring URL entry {domain} in allowed_domains."
)
warnings.warn(message)
warnings.warn(message, stacklevel=2)
elif port_pattern.search(domain):
message = (
"allowed_domains accepts only domains without ports. "
f"Ignoring entry {domain} in allowed_domains."
)
warnings.warn(message)
warnings.warn(message, stacklevel=2)
else:
domains.append(re.escape(domain))
regex = rf"^(.*\.)?({'|'.join(domains)})$"

19
scrapy/downloadermiddlewares/redirect.py
View File

@ -71,7 +71,7 @@ class BaseRedirectMiddleware:
return
redirect_cls = global_object_name(self.__class__)
referer_cls = global_object_name(RefererMiddleware)
if self.__class__ in (RedirectMiddleware, MetaRefreshMiddleware):
if self.__class__ in {RedirectMiddleware, MetaRefreshMiddleware}:
replacement = (
f"replace {redirect_cls} with a subclass that overrides the "
f"handle_referer() method"
@ -208,14 +208,19 @@ class RedirectMiddleware(BaseRedirectMiddleware):
if (
request.meta.get("dont_redirect", False)
or response.status
in getattr(self.crawler.spider, "handle_httpstatus_list", [])
or response.status in request.meta.get("handle_httpstatus_list", [])
in getattr(self.crawler.spider, "handle_httpstatus_list", ())
or response.status in request.meta.get("handle_httpstatus_list", ())
or request.meta.get("handle_httpstatus_all", False)
):
return response
allowed_status = (301, 302, 303, 307, 308)
if "Location" not in response.headers or response.status not in allowed_status:
if "Location" not in response.headers or response.status not in {
301,
302,
303,
307,
308,
}:
return response
assert response.headers["Location"] is not None
@ -235,8 +240,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
if urlparse_cached(redirected).scheme not in {"http", "https"}:
return response
if (response.status in (301, 302) and request.method == "POST") or (
response.status == 303 and request.method not in ("GET", "HEAD")
if (response.status in {301, 302} and request.method == "POST") or (
response.status == 303 and request.method not in {"GET", "HEAD"}
):
redirected = self._redirect_request_using_get(
request, response, redirected_url

31
scrapy/extensions/feedexport.py
View File

@ -200,7 +200,7 @@ class S3FeedStorage(BlockingFeedStorage):
try:
import boto3.session # noqa: PLC0415
except ImportError:
raise NotConfigured("missing boto3 library")
raise NotConfigured("missing boto3 library") from None
u = urlparse(uri)
assert u.hostname
self.bucketname: str = u.hostname
@ -250,10 +250,19 @@ class S3FeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file: IO[bytes]) -> None:
file.seek(0)
kwargs: dict[str, Any] = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
self.s3_client.upload_fileobj(
Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
)
if self.acl:
self.s3_client.upload_fileobj(
Bucket=self.bucketname,
Key=self.keyname,
Fileobj=file,
ExtraArgs={"ACL": self.acl},
)
else:
self.s3_client.upload_fileobj(
Bucket=self.bucketname,
Key=self.keyname,
Fileobj=file,
)
file.close()
@ -468,9 +477,13 @@ class FeedExporter:
# End: Backward compatibility for FEED_URI and FEED_FORMAT settings
# 'FEEDS' setting takes precedence over 'FEED_URI'
for uri, feed_options in self.settings.getdict("FEEDS").items():
for settings_uri, feed_options in self.settings.getdict("FEEDS").items():
# handle pathlib.Path objects
uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
uri = (
str(settings_uri)
if not isinstance(settings_uri, Path)
else settings_uri.absolute().as_uri()
)
self.feeds[uri] = feed_complete_default_values_from_settings(
feed_options, self.settings
)
@ -692,9 +705,7 @@ class FeedExporter:
uri_params_function: str | UriParamsCallableT | None,
slot: FeedSlot | None = None,
) -> dict[str, Any]:
params = {}
for k in dir(spider):
params[k] = getattr(spider, k)
params = {k: getattr(spider, k) for k in dir(spider)}
utc_now = datetime.now(tz=timezone.utc)
params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-")
params["batch_time"] = utc_now.isoformat().replace(":", "-")

6
scrapy/extensions/httpcache.py
View File

@ -106,10 +106,10 @@ class RFC2616Policy:
if b"max-age" in cc or b"Expires" in response.headers:
return True
# Firefox fallbacks this statuses to one year expiration if none is set
if response.status in (300, 301, 308):
if response.status in {300, 301, 308}:
return True
# Other statuses without expiration requires at least one validator
if response.status in (200, 203, 401):
if response.status in {200, 203, 401}:
return b"Last-Modified" in response.headers or b"ETag" in response.headers
# Any other is probably not eligible for caching
# Makes no sense to cache responses that does not contain expiration
@ -216,7 +216,7 @@ class RFC2616Policy:
return (date - lastmodified) / 10
# This request can be cached indefinitely
if response.status in (300, 301, 308):
if response.status in {300, 301, 308}:
return self.MAXAGE
# Insufficient information to compute freshness lifetime

4
scrapy/extensions/memusage.py
View File

@ -39,8 +39,8 @@ class MemoryUsage:
try:
# stdlib's resource module is only available on unix platforms.
self.resource = import_module("resource")
except ImportError:
raise NotConfigured
except ImportError as exc:
raise NotConfigured from exc
self.crawler: Crawler = crawler
self.warned: bool = False

23
scrapy/extensions/periodic_log.py
View File

@ -10,6 +10,7 @@ from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
from scrapy.utils.serialize import ScrapyJSONEncoder
if TYPE_CHECKING:
from collections.abc import Sequence
from json import JSONEncoder
from twisted.internet.task import LoopingCall
@ -31,8 +32,8 @@ class PeriodicLog:
self,
stats: StatsCollector,
interval: float = 60.0,
ext_stats: dict[str, Any] = {},
ext_delta: dict[str, Any] = {},
ext_stats: dict[str, Any] | None = None,
ext_delta: dict[str, Any] | None = None,
ext_timing_enabled: bool = False,
):
self.stats: StatsCollector = stats
@ -41,11 +42,19 @@ class PeriodicLog:
self.task: AsyncioLoopingCall | LoopingCall | None = None
self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
self.ext_stats_enabled: bool = bool(ext_stats)
self.ext_stats_include: list[str] = ext_stats.get("include", [])
self.ext_stats_exclude: list[str] = ext_stats.get("exclude", [])
self.ext_stats_include: Sequence[str] = (
ext_stats.get("include", ()) if ext_stats else ()
)
self.ext_stats_exclude: Sequence[str] = (
ext_stats.get("exclude", ()) if ext_stats else ()
)
self.ext_delta_enabled: bool = bool(ext_delta)
self.ext_delta_include: list[str] = ext_delta.get("include", [])
self.ext_delta_exclude: list[str] = ext_delta.get("exclude", [])
self.ext_delta_include: Sequence[str] = (
ext_delta.get("include", ()) if ext_delta else ()
)
self.ext_delta_exclude: Sequence[str] = (
ext_delta.get("exclude", ()) if ext_delta else ()
)
self.ext_timing_enabled: bool = ext_timing_enabled
@classmethod
@ -143,7 +152,7 @@ class PeriodicLog:
return {"stats": stats}
def param_allowed(
self, stat_name: str, include: list[str], exclude: list[str]
self, stat_name: str, include: Sequence[str], exclude: Sequence[str]
) -> bool:
if not include and not exclude:
return True

1
scrapy/extensions/statsmailer.py
View File

@ -25,6 +25,7 @@ if TYPE_CHECKING:
warnings.warn(
"The scrapy.extensions.statsmailer module is deprecated and will be "
"removed in a future release.",
stacklevel=2,
category=ScrapyDeprecationWarning,
)

4
scrapy/http/cookies.py
View File

@ -54,14 +54,14 @@ class CookieJar:
if not IPV4_RE.search(req_host):
hosts = potential_domain_matches(req_host)
if "." not in req_host:
hosts += [req_host + ".local"]
hosts.append(req_host + ".local")
else:
hosts = [req_host]
cookies = []
for host in hosts:
if host in self.jar._cookies: # type: ignore[attr-defined]
cookies += self.jar._cookies_for_domain(host, wreq) # type: ignore[attr-defined]
cookies.extend(self.jar._cookies_for_domain(host, wreq)) # type: ignore[attr-defined]
attrs = self.jar._cookie_attrs(cookies) # type: ignore[attr-defined]
if attrs and not wreq.has_header("Cookie"):

8
scrapy/http/request/form.py
View File

@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, TypeAlias, cast
from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
from parsel.csstranslator import HTMLTranslator
@ -39,7 +39,7 @@ FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None
class FormRequest(Request):
__slots__ = ()
valid_form_methods = ["GET", "POST"]
valid_form_methods: ClassVar[list[str]] = ["GET", "POST"]
def __init__(
self, *args: Any, formdata: FormdataType = None, **kwargs: Any
@ -153,7 +153,7 @@ def _get_form(
try:
form = forms[formnumber]
except IndexError:
raise IndexError(f"Form number {formnumber} not found in {response}")
raise IndexError(f"Form number {formnumber} not found in {response}") from None
return cast("FormElement", form)
@ -167,7 +167,7 @@ def _get_inputs(
try:
formdata_keys = dict(formdata or ()).keys()
except (ValueError, TypeError):
raise ValueError("formdata should be a dict or iterable of tuples")
raise ValueError("formdata should be a dict or iterable of tuples") from None
if not formdata:
formdata = []

8
scrapy/http/request/json_request.py
View File

@ -36,7 +36,9 @@ class JsonRequest(Request):
data_passed: bool = data is not None
if body_passed and data_passed:
warnings.warn("Both body and data passed. data will be ignored")
warnings.warn(
"Both body and data passed. data will be ignored", stacklevel=2
)
elif not body_passed and data_passed:
kwargs["body"] = self._dumps(data)
if "method" not in kwargs:
@ -68,7 +70,9 @@ class JsonRequest(Request):
data_passed: bool = data is not None
if body_passed and data_passed:
warnings.warn("Both body and data passed. data will be ignored")
warnings.warn(
"Both body and data passed. data will be ignored", stacklevel=2
)
elif not body_passed and data_passed:
kwargs["body"] = self._dumps(data)

4
scrapy/http/response/__init__.py
View File

@ -86,7 +86,7 @@ class Response(object_ref):
raise AttributeError(
"Response.cb_kwargs not available, this response "
"is not tied to any request"
)
) from None
@property
def meta(self) -> dict[str, Any]:
@ -95,7 +95,7 @@ class Response(object_ref):
except AttributeError:
raise AttributeError(
"Response.meta not available, this response is not tied to any request"
)
) from None
@property
def url(self) -> str:

2
scrapy/http/response/text.py
View File

@ -297,7 +297,7 @@ def _url_from_selector(sel: parsel.Selector) -> str:
return strip_html5_whitespace(sel.root)
if not hasattr(sel.root, "tag"):
raise _InvalidSelector(f"Unsupported selector: {sel}")
if sel.root.tag not in ("a", "link"):
if sel.root.tag not in {"a", "link"}:
raise _InvalidSelector(
f"Only <a> and <link> elements are supported; got <{sel.root.tag}>"
)

6
scrapy/linkextractors/lxmlhtml.py
View File

@ -114,8 +114,8 @@ class LxmlParserLinkExtractor:
# pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
try:
if self.strip:
attr_val = strip_html5_whitespace(attr_val)
attr_val = urljoin(base_url, attr_val)
attr_val = strip_html5_whitespace(attr_val) # noqa: PLW2901 this is intended
attr_val = urljoin(base_url, attr_val) # noqa: PLW2901
except ValueError:
continue # skipping bogus links
else:
@ -245,7 +245,7 @@ class LxmlLinkExtractor:
if self.allow_res
else [True]
)
denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else []
denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else ()
return any(allowed) and not any(denied)
def _process_links(self, links: list[Link]) -> list[Link]:

1
scrapy/mail.py
View File

@ -55,6 +55,7 @@ def _to_bytes_or_none(text: str | bytes | None) -> bytes | None:
warnings.warn(
"The scrapy.mail module is deprecated and will be removed in a future release. "
"Please use a dedicated Python mail library instead.",
stacklevel=2,
category=ScrapyDeprecationWarning,
)

16
scrapy/pipelines/files.py
View File

@ -18,7 +18,7 @@ from contextlib import suppress
from ftplib import FTP
from io import BytesIO
from pathlib import Path
from typing import IO, TYPE_CHECKING, Any, NoReturn, Protocol, TypedDict, cast
from typing import IO, TYPE_CHECKING, Any, ClassVar, NoReturn, Protocol, TypedDict, cast
from urllib.parse import urlparse
from itemadapter import ItemAdapter
@ -161,7 +161,7 @@ class S3FilesStore:
AWS_VERIFY = None
POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler()
HEADERS = {
HEADERS: ClassVar[dict[str, str]] = {
"Cache-Control": "max-age=172800",
}
@ -226,7 +226,7 @@ class S3FilesStore:
Bucket=self.bucket,
Key=key_name,
Body=buf,
Metadata={k: str(v) for k, v in (meta or {}).items()},
Metadata={k: str(v) for k, v in meta.items()} if meta else {},
ACL=self.POLICY,
**extra,
)
@ -269,7 +269,9 @@ class S3FilesStore:
try:
kwarg = mapping[key]
except KeyError:
raise TypeError(f'Header "{key}" is not supported by botocore')
raise TypeError(
f'Header "{key}" is not supported by botocore'
) from None
extra[kwarg] = value
return extra
@ -339,7 +341,7 @@ class GCSFilesStore:
blob_path = self._get_blob_path(path)
blob = self.bucket.blob(blob_path)
blob.cache_control = self.CACHE_CONTROL
blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
blob.metadata = {k: str(v) for k, v in meta.items()} if meta else {}
return deferred_from_coro(
run_in_thread(
blob.upload_from_string,
@ -435,7 +437,7 @@ class FilesPipeline(MediaPipeline):
MEDIA_NAME: str = "file"
EXPIRES: int = 90
STORE_SCHEMES: dict[str, type[FilesStoreProtocol]] = {
STORE_SCHEMES: ClassVar[dict[str, type[FilesStoreProtocol]]] = {
"": FSFilesStore,
"file": FSFilesStore,
"s3": S3FilesStore,
@ -656,7 +658,7 @@ class FilesPipeline(MediaPipeline):
exc_info=True,
extra={"spider": info.spider},
)
raise FileException(str(exc))
raise FileException(str(exc)) from exc
return {
"url": request.url,

8
scrapy/pipelines/images.py
View File

@ -11,7 +11,7 @@ import hashlib
import warnings
from contextlib import suppress
from io import BytesIO
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, ClassVar
from itemadapter import ItemAdapter
@ -49,7 +49,7 @@ class ImagesPipeline(FilesPipeline):
MIN_WIDTH: int = 0
MIN_HEIGHT: int = 0
EXPIRES: int = 90
THUMBS: dict[str, tuple[int, int]] = {}
THUMBS: ClassVar[dict[str, tuple[int, int]]] = {}
DEFAULT_IMAGES_URLS_FIELD = "image_urls"
DEFAULT_IMAGES_RESULT_FIELD = "images"
@ -76,7 +76,7 @@ class ImagesPipeline(FilesPipeline):
except ImportError:
raise NotConfigured(
"ImagesPipeline requires installing Pillow 8.3.2 or later"
)
) from None
super().__init__(store_uri, crawler=crawler)
@ -191,7 +191,7 @@ class ImagesPipeline(FilesPipeline):
*,
response_body: BytesIO,
) -> tuple[Image.Image, BytesIO]:
if image.format in ("PNG", "WEBP") and image.mode == "RGBA":
if image.format in {"PNG", "WEBP"} and image.mode == "RGBA":
background = self._Image.new("RGBA", image.size, (255, 255, 255))
background.paste(image, image)
image = background.convert("RGB")

4
scrapy/pipelines/media.py
View File

@ -230,9 +230,9 @@ class MediaPipeline(ABC):
if isinstance(result, Failure):
# minimize cached information for failure
result.cleanFailure()
result.frames = []
result.frames.clear()
if TWISTED_FAILURE_HAS_STACK:
result.stack = [] # type: ignore[method-assign]
result.stack.clear()
# This code fixes a memory leak by avoiding to keep references to
# the Request and Response objects on the Media Pipeline cache.
#

5
scrapy/pqueues.py
View File

@ -350,8 +350,9 @@ class DownloaderAwarePriorityQueue:
self.crawler: Crawler = crawler
self.pqueues: dict[str, ScrapyPriorityQueue] = {} # slot -> priority queue
for slot, startprios in (slot_startprios or {}).items():
self.pqueues[slot] = self.pqfactory(slot, startprios)
if slot_startprios:
for slot, startprios in slot_startprios.items():
self.pqueues[slot] = self.pqfactory(slot, startprios)
def pqfactory(
self, slot: str, startprios: Iterable[int] = ()

4
scrapy/responsetypes.py
View File

@ -8,7 +8,7 @@ from __future__ import annotations
from io import StringIO
from mimetypes import MimeTypes
from pkgutil import get_data
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, ClassVar
from scrapy.http import Response
from scrapy.utils.misc import load_object
@ -19,7 +19,7 @@ if TYPE_CHECKING:
class ResponseTypes:
CLASSES = {
CLASSES: ClassVar[dict[str, str]] = {
"text/html": "scrapy.http.HtmlResponse",
"application/atom+xml": "scrapy.http.XmlResponse",
"application/rdf+xml": "scrapy.http.XmlResponse",

6
scrapy/settings/__init__.py
View File

@ -184,15 +184,15 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
try:
return bool(int(got))
except ValueError:
if got in ("True", "true"):
if got in {"True", "true"}:
return True
if got in ("False", "false"):
if got in {"False", "false"}:
return False
raise ValueError(
"Supported values for boolean settings "
"are 0/1, True/False, '0'/'1', "
"'True'/'False' and 'true'/'false'"
)
) from None
def getint(self, name: _SettingsKey, default: int = 0) -> int:
"""

4
scrapy/spiderloader.py
View File

@ -77,6 +77,7 @@ class SpiderLoader:
warnings.warn(
"There are several spiders with the same name:\n\n"
f"{dupes_string}\n\n This can cause unexpected behavior.",
stacklevel=2,
category=UserWarning,
)
@ -96,6 +97,7 @@ class SpiderLoader:
f"\n{traceback.format_exc()}Could not load spiders "
f"from module '{name}'. "
"See above traceback for details.",
stacklevel=2,
category=RuntimeWarning,
)
else:
@ -114,7 +116,7 @@ class SpiderLoader:
try:
return self._spiders[spider_name]
except KeyError:
raise KeyError(f"Spider not found: {spider_name}")
raise KeyError(f"Spider not found: {spider_name}") from None
def find_by_request(self, request: Request) -> list[str]:
"""

2
scrapy/spidermiddlewares/httperror.py
View File

@ -88,5 +88,5 @@ class HttpErrorMiddleware:
{"response": response},
extra={"spider": self.crawler.spider},
)
return []
return ()
return None

4
scrapy/spidermiddlewares/referer.py
View File

@ -103,7 +103,7 @@ class ReferrerPolicy(ABC):
return self.tls_protected(url)
def tls_protected(self, url: str) -> bool:
return urlparse(url).scheme in ("https", "ftps")
return urlparse(url).scheme in {"https", "ftps"}
class NoReferrerPolicy(ReferrerPolicy):
@ -424,7 +424,7 @@ class RefererMiddleware(BaseSpiderMiddleware):
msg += " (import paths from the response Referrer-Policy header are not allowed)"
if not warning_only:
raise RuntimeError(msg)
warnings.warn(msg, RuntimeWarning)
warnings.warn(msg, RuntimeWarning, stacklevel=2)
return None
def get_processed_request(

16
scrapy/spiders/crawl.py
View File

@ -102,11 +102,12 @@ class CrawlSpider(Spider):
self._compile_rules()
if method_is_overridden(self.__class__, CrawlSpider, "_parse_response"):
warnings.warn(
f"The CrawlSpider._parse_response method, which the "
"The CrawlSpider._parse_response method, which the "
f"{global_object_name(self.__class__)} class overrides, is "
f"deprecated: it will be removed in future Scrapy releases. "
f"Please override the CrawlSpider.parse_with_rules method "
f"instead."
"deprecated: it will be removed in future Scrapy releases. "
"Please override the CrawlSpider.parse_with_rules method "
"instead.",
stacklevel=2,
)
def _parse(self, response: Response, **kwargs: Any) -> Any:
@ -118,7 +119,7 @@ class CrawlSpider(Spider):
)
def parse_start_url(self, response: Response, **kwargs: Any) -> Any:
return []
return ()
def process_results(
self, response: Response, results: Iterable[Any]
@ -209,8 +210,9 @@ class CrawlSpider(Spider):
def _compile_rules(self) -> None:
self._rules = []
for rule in self.rules:
self._rules.append(copy.copy(rule))
self._rules[-1]._compile(self)
copied_rule = copy.copy(rule)
copied_rule._compile(self)
self._rules.append(copied_rule)
@classmethod
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:

2
scrapy/spiders/sitemap.py
View File

@ -49,7 +49,7 @@ class SitemapSpider(Spider):
self._cbs: list[tuple[re.Pattern[str], CallbackT]] = []
for r, c in self.sitemap_rules:
if isinstance(c, str):
c = cast("CallbackT", getattr(self, c))
c = cast("CallbackT", getattr(self, c)) # noqa: PLW2901
self._cbs.append((regex(r), c))
self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]

16
scrapy/utils/console.py
View File

@ -13,7 +13,7 @@ KnownShellsT = dict[str, Callable[..., EmbedFuncT]]
def _embed_ipython_shell(
namespace: dict[str, Any] = {}, banner: str = ""
namespace: dict[str, Any] | None = None, banner: str = ""
) -> EmbedFuncT:
"""Start an IPython Shell"""
try:
@ -28,7 +28,7 @@ def _embed_ipython_shell(
)
@wraps(_embed_ipython_shell)
def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
config = load_default_config() # type: ignore[no-untyped-call]
# Always use .instance() to ensure _instance propagation to all parents
# this is needed for <TAB> completion works well for new imports
@ -44,26 +44,26 @@ def _embed_ipython_shell(
def _embed_bpython_shell(
namespace: dict[str, Any] = {}, banner: str = ""
namespace: dict[str, Any] | None = None, banner: str = ""
) -> EmbedFuncT:
"""Start a bpython shell"""
import bpython # noqa: PLC0415
@wraps(_embed_bpython_shell)
def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
bpython.embed(locals_=namespace, banner=banner)
return wrapper
def _embed_ptpython_shell(
namespace: dict[str, Any] = {}, banner: str = ""
namespace: dict[str, Any] | None = None, banner: str = ""
) -> EmbedFuncT:
"""Start a ptpython shell"""
import ptpython.repl # noqa: PLC0415 # pylint: disable=import-error
@wraps(_embed_ptpython_shell)
def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
print(banner)
ptpython.repl.embed(locals=namespace)
@ -71,7 +71,7 @@ def _embed_ptpython_shell(
def _embed_standard_shell(
namespace: dict[str, Any] = {}, banner: str = ""
namespace: dict[str, Any] | None = None, banner: str = ""
) -> EmbedFuncT:
"""Start a standard python shell"""
try: # readline module is only available on unix systems
@ -84,7 +84,7 @@ def _embed_standard_shell(
readline.parse_and_bind("tab:complete") # type: ignore[attr-defined,unused-ignore]
@wraps(_embed_standard_shell)
def wrapper(namespace: dict[str, Any] = namespace, banner: str = "") -> None:
def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
code.interact(banner=banner, local=namespace)
return wrapper

2
scrapy/utils/curl.py
View File

@ -106,7 +106,7 @@ def curl_to_request_kwargs(
if argv:
msg = f"Unrecognized options: {', '.join(argv)}"
if ignore_unknown_options:
warnings.warn(msg)
warnings.warn(msg, stacklevel=2)
else:
raise ValueError(msg)

5
scrapy/utils/defer.py
View File

@ -8,6 +8,7 @@ import asyncio
import inspect
import warnings
from asyncio import Future
from collections import deque
from collections.abc import Awaitable, Coroutine, Iterable, Iterator
from functools import wraps
from typing import (
@ -230,7 +231,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
self.callable_args: tuple[Any, ...] = callable_args
self.callable_kwargs: dict[str, Any] = callable_kwargs
self.finished: bool = False
self.waiting_deferreds: list[Deferred[Any]] = []
self.waiting_deferreds: deque[Deferred[Any]] = deque()
self.anext_deferred: Deferred[_T] | None = None
def _callback(self, result: _T) -> None:
@ -241,7 +242,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
callable_result = self.callable(
result, *self.callable_args, **self.callable_kwargs
)
d = self.waiting_deferreds.pop(0)
d = self.waiting_deferreds.popleft()
if isinstance(callable_result, Deferred):
callable_result.chainDeferred(d)
else:

3
scrapy/utils/deprecate.py
View File

@ -130,7 +130,7 @@ def create_deprecated_class(
# deprecated class is in jinja2 template). __module__ attribute is not
# important enough to raise an exception as users may be unable
# to fix inspect.stack() errors.
warnings.warn(f"Error detecting parent module: {e!r}")
warnings.warn(f"Error detecting parent module: {e!r}", stacklevel=2)
return deprecated_cls
@ -160,6 +160,7 @@ def update_classpath(path: Any) -> Any:
warnings.warn(
f"`{path}` class is deprecated, use `{new_path}` instead",
ScrapyDeprecationWarning,
stacklevel=2,
)
return new_path
return path

64
scrapy/utils/misc.py
View File

@ -13,7 +13,7 @@ from contextlib import contextmanager
from functools import partial
from importlib import import_module
from pkgutil import iter_modules
from typing import IO, TYPE_CHECKING, Any, TypeVar, cast
from typing import IO, TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, overload
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.item import Item
@ -28,9 +28,20 @@ if TYPE_CHECKING:
_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
T = TypeVar("T")
_ITER_T = TypeVar("_ITER_T", bound=dict | Item | str | bytes)
_T = TypeVar("_T")
_T_co = TypeVar("_T_co", covariant=True)
_P = ParamSpec("_P")
@overload
def arg_to_iter(arg: None) -> tuple[()]: ...
@overload
def arg_to_iter(arg: _ITER_T) -> Iterable[_ITER_T]: ...
@overload
def arg_to_iter(arg: Iterable[_T]) -> Iterable[_T]: ...
@overload
def arg_to_iter(arg: _T) -> Iterable[_T]: ...
def arg_to_iter(arg: Any) -> Iterable[Any]:
"""Convert an argument to an iterable. The argument can be a None, single
value, or an iterable.
@ -38,9 +49,9 @@ def arg_to_iter(arg: Any) -> Iterable[Any]:
Exception: if arg is a dict, [arg] will be returned
"""
if arg is None:
return []
return ()
if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):
return cast("Iterable[Any]", arg)
return arg
return [arg]
@ -64,7 +75,7 @@ def load_object(path: str | Callable[..., Any]) -> Any:
try:
dot = path.rindex(".")
except ValueError:
raise ValueError(f"Error loading object '{path}': not a full path")
raise ValueError(f"Error loading object '{path}': not a full path") from None
module, name = path[:dot], path[dot + 1 :]
mod = import_module(module)
@ -72,7 +83,9 @@ def load_object(path: str | Callable[..., Any]) -> Any:
try:
obj = getattr(mod, name)
except AttributeError:
raise NameError(f"Module '{module}' doesn't define any object named '{name}'")
raise NameError(
f"Module '{module}' doesn't define any object named '{name}'"
) from None
return obj
@ -152,9 +165,40 @@ def rel_has_nofollow(rel: str | None) -> bool:
return rel is not None and "nofollow" in rel.replace(",", " ").split()
class SupportsFromCrawler(Protocol[_T_co, _P]):
@classmethod
def from_crawler(
cls, crawler: Crawler, /, *args: _P.args, **kwargs: _P.kwargs
) -> _T_co: ...
@overload
def build_from_crawler(
objcls: type[T], crawler: Crawler, /, *args: Any, **kwargs: Any
) -> T:
objcls: SupportsFromCrawler[_T_co, _P],
crawler: Crawler,
/,
*args: _P.args,
**kwargs: _P.kwargs,
) -> _T_co: ...
@overload
def build_from_crawler(
objcls: Callable[_P, _T_co],
crawler: Crawler,
/,
*args: _P.args,
**kwargs: _P.kwargs,
) -> _T_co: ...
def build_from_crawler(
objcls: Any,
crawler: Crawler,
/,
*args: Any,
**kwargs: Any,
) -> Any:
"""Construct a class instance using its ``from_crawler()`` or ``__init__()`` constructor.
.. versionadded:: 2.12
@ -164,14 +208,14 @@ def build_from_crawler(
Raises ``TypeError`` if the resulting instance is ``None``.
"""
if hasattr(objcls, "from_crawler"):
instance = objcls.from_crawler(crawler, *args, **kwargs) # type: ignore[attr-defined]
instance = objcls.from_crawler(crawler, *args, **kwargs)
method_name = "from_crawler"
else:
instance = objcls(*args, **kwargs)
method_name = "__new__"
if instance is None:
raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
return cast("T", instance)
return instance
@contextmanager

3
scrapy/utils/project.py
View File

@ -20,7 +20,8 @@ def inside_project() -> bool:
import_module(scrapy_module)
except ImportError as exc:
warnings.warn(
f"Cannot import scrapy settings module {scrapy_module}: {exc}"
f"Cannot import scrapy settings module {scrapy_module}: {exc}",
stacklevel=2,
)
else:
return True

2
scrapy/utils/reactor.py
View File

@ -76,7 +76,7 @@ class CallLaterOnce(Generic[_T]):
for d in self._deferreds:
call_later(0, d.callback, None)
self._deferreds = []
self._deferreds.clear()
return result

2
scrapy/utils/request.py
View File

@ -169,7 +169,7 @@ def _get_method(obj: Any, name: Any) -> Any:
try:
return getattr(obj, name)
except AttributeError:
raise ValueError(f"Method {name!r} not found in: {obj}")
raise ValueError(f"Method {name!r} not found in: {obj}") from None
def request_to_curl(request: Request) -> str:

5
scrapy/utils/testproc.py
View File

@ -4,7 +4,7 @@ from __future__ import annotations
import os
import sys
import warnings
from typing import TYPE_CHECKING, cast
from typing import TYPE_CHECKING, ClassVar, cast
from twisted.internet.defer import Deferred
from twisted.internet.protocol import ProcessProtocol
@ -21,12 +21,13 @@ if TYPE_CHECKING:
warnings.warn(
"The scrapy.utils.testproc module is deprecated.",
ScrapyDeprecationWarning,
stacklevel=2,
)
class ProcessTest:
command: str | None = None
prefix = [sys.executable, "-m", "scrapy.cmdline"]
prefix: ClassVar[list[str]] = [sys.executable, "-m", "scrapy.cmdline"]
cwd = os.getcwd() # trial chdirs to temp dir # noqa: PTH109
def execute(

1
scrapy/utils/testsite.py
View File

@ -9,6 +9,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
warnings.warn(
"The scrapy.utils.testsite module is deprecated.",
ScrapyDeprecationWarning,
stacklevel=2,
)

24
scrapy/utils/url.py
View File

@ -19,13 +19,18 @@ from w3lib.url import parse_url as _parse_url
from scrapy.exceptions import ScrapyDeprecationWarning
_DEPRECATED_NAMES: frozenset[str] = frozenset(
{"_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects}
)
def __getattr__(name: str) -> Any:
if name in ("_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects):
if name in _DEPRECATED_NAMES:
obj_type = "attribute" if name == "_safe_chars" else "function"
warnings.warn(
f"The scrapy.utils.url.{name} {obj_type} is deprecated, use w3lib.url.{name} instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return getattr(import_module("w3lib.url"), name)
@ -45,15 +50,18 @@ def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:
host = _parse_url(url).netloc.lower()
if not host:
return False
domains = [d.lower() for d in domains]
return any((host == d) or (host.endswith(f".{d}")) for d in domains)
return any((host == d) or (host.endswith(f".{d}")) for d in map(str.lower, domains))
def _spider_domains(spider: type[Spider]) -> Iterable[str]:
yield spider.name
if allowed_domains := getattr(spider, "allowed_domains", None):
yield from allowed_domains
def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool:
"""Return True if the url belongs to the given spider"""
return url_is_from_any_domain(
url, [spider.name, *getattr(spider, "allowed_domains", [])]
)
return url_is_from_any_domain(url, _spider_domains(spider))
def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:
@ -184,11 +192,11 @@ def strip_url(
strip_default_port
and parsed_url.port
and (parsed_url.scheme, parsed_url.port)
in (
in {
("http", 80),
("https", 443),
("ftp", 21),
)
}
):
netloc = netloc.replace(f":{parsed_url.port}", "")

2
tests/test_spidermiddleware_httperror.py
View File

@ -88,7 +88,7 @@ class TestHttpErrorMiddleware:
def test_process_spider_exception(
self, mw: HttpErrorMiddleware, res404: Response
) -> None:
assert mw.process_spider_exception(res404, HttpError(res404)) == []
assert mw.process_spider_exception(res404, HttpError(res404)) == ()
assert mw.process_spider_exception(res404, Exception()) is None
def test_handle_httpstatus_list(