RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Bump ruff, fix some of rules (#7277)

This commit is contained in:
Andrey Rakhmatullin 2026-02-23 15:48:38 +05:00 committed by GitHub
parent abd025f78e
commit 7010985e4f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
30 changed files with 70 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.pre-commit-config.yaml
View File

@ -6,7 +6,7 @@ exclude: |
)
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.14.2
rev: v0.15.2
hooks:
- id: ruff-check
args: [ --fix ]

41
pyproject.toml
View File

@ -174,7 +174,6 @@ disable = [
"disallowed-name",
"duplicate-code", # https://github.com/pylint-dev/pylint/issues/214
"fixme",
"import-outside-toplevel",
"inherit-non-class", # false positives with create_deprecated_class()
"invalid-name",
"invalid-overridden-method",
@ -188,12 +187,10 @@ disable = [
"no-value-for-parameter", # https://github.com/pylint-dev/pylint/issues/3268
"not-callable",
"protected-access",
"redefined-builtin",
"redefined-outer-name",
"too-few-public-methods",
"too-many-ancestors",
"too-many-arguments",
"too-many-branches",
"too-many-function-args",
"too-many-instance-attributes",
"too-many-lines",
@ -202,12 +199,22 @@ disable = [
"too-many-public-methods",
"too-many-return-statements",
"unused-argument",
"unused-import",
"unused-variable",
"useless-import-alias", # used as a hint to mypy
"useless-return", # https://github.com/pylint-dev/pylint/issues/6530
"wrong-import-position",
# Ones that are implemented in ruff and need to be disabled for some lines (listed here to avoid two disable comments)
"bare-except",
"eval-used",
"global-statement",
"import-outside-toplevel",
"import-self",
"inconsistent-return-statements",
"redefined-builtin",
"too-many-branches",
"unused-import",
# Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
"abstract-method",
"arguments-differ",
@ -345,34 +352,22 @@ ignore = [
"D403",
# `try`-`except` within a loop incurs performance overhead
"PERF203",
# Import alias does not rename original package
"PLC0414",
# Too many return statements
"PLR0911",
# Too many branches
"PLR0912",
# Too many arguments in function definition
"PLR0913",
# Too many statements
"PLR0915",
# Magic value used in comparison
"PLR2004",
# `for` loop variable overwritten by assignment target
"PLW2901",
# String contains ambiguous {}.
"RUF001",
# Docstring contains ambiguous {}.
"RUF002",
# Comment contains ambiguous {}.
"RUF003",
# Mutable class attributes should be annotated with `typing.ClassVar`
"RUF012",
# Use of `assert` detected; needed for mypy
"S101",
# FTP-related functions are being called; https://github.com/scrapy/scrapy/issues/4180
"S321",
# Argument default set to insecure SSL protocol
"S503",
# Use a context manager for opening files
"SIM115",
# Yoda condition detected
@ -384,18 +379,16 @@ ignore = [
"B003",
# Do not use mutable data structures for argument defaults.
"B006",
# Loop control variable not used within the loop body.
"B007",
# Do not perform function calls in argument defaults.
"B008",
# Found useless expression.
"B018",
# Star-arg unpacking after a keyword argument is strongly discouraged.
"B026",
# No explicit stacklevel argument found.
"B028",
# Within an `except` clause, raise exceptions with `raise ... from`
"B904",
# `for` loop variable overwritten by assignment target
"PLW2901",
# Mutable class attributes should be annotated with `typing.ClassVar`
"RUF012",
# Use capitalized environment variable
"SIM112",
]
@ -417,8 +410,8 @@ split-on-trailing-comma = false
"scrapy/linkextractors/__init__.py" = ["E402"]
"scrapy/spiders/__init__.py" = ["E402"]
# Skip bandit in tests
"tests/**" = ["S"]
# Skip bandit and allow blocking file I/O in tests
"tests/**" = ["ASYNC240", "S"]
# Issues pending a review:
"docs/conf.py" = ["E402"]

4
scrapy/core/downloader/__init__.py
View File

@ -157,9 +157,7 @@ class Downloader:
if key not in self.slots:
assert self.crawler.spider
slot_settings = self.per_slot_settings.get(key, {})
conc = (
self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
)
conc = self.ip_concurrency or self.domain_concurrency
conc, delay = _get_concurrency_delay(
conc, self.crawler.spider, self.settings
)

6
scrapy/core/downloader/contextfactory.py
View File

@ -49,7 +49,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
def __init__(
self,
method: int = SSL.SSLv23_METHOD,
method: int = SSL.SSLv23_METHOD, # noqa: S503
tls_verbose_logging: bool = False,
tls_ciphers: str | None = None,
*args: Any,
@ -75,7 +75,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
def from_crawler(
cls,
crawler: Crawler,
method: int = SSL.SSLv23_METHOD,
method: int = SSL.SSLv23_METHOD, # noqa: S503
*args: Any,
**kwargs: Any,
) -> Self:
@ -84,10 +84,10 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
)
tls_ciphers: str | None = crawler.settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
return cls( # type: ignore[misc]
*args,
method=method,
tls_verbose_logging=tls_verbose_logging,
tls_ciphers=tls_ciphers,
*args,
**kwargs,
)

2
scrapy/core/downloader/handlers/file.py
View File

@ -16,6 +16,6 @@ if TYPE_CHECKING:
class FileDownloadHandler(BaseDownloadHandler):
async def download_request(self, request: Request) -> Response:
filepath = file_uri_to_path(request.url)
body = Path(filepath).read_bytes()
body = Path(filepath).read_bytes() # noqa: ASYNC240
respcls = responsetypes.from_args(filename=filepath, body=body)
return respcls(url=request.url, body=body)

4
scrapy/core/engine.py
View File

@ -439,7 +439,7 @@ class ExecutionEngine:
spider=self.spider,
dont_log=IgnoreRequest,
)
for handler, result in request_scheduled_result:
for _, result in request_scheduled_result:
if isinstance(result, Failure) and isinstance(result.value, IgnoreRequest):
return
if not self._slot.scheduler.enqueue_request(request): # type: ignore[union-attr]
@ -587,7 +587,7 @@ class ExecutionEngine:
)
return deferred_from_coro(self.close_spider_async(reason=reason))
async def close_spider_async(self, *, reason: str = "cancelled") -> None:
async def close_spider_async(self, *, reason: str = "cancelled") -> None: # noqa: PLR0912
"""Close (cancel) spider and clear all its outstanding requests.
.. versionadded:: 2.14

4
scrapy/core/http2/agent.py
View File

@ -120,7 +120,7 @@ class H2Agent:
self,
reactor: ReactorBase,
pool: H2ConnectionPool,
context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),
context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), # noqa: B008
connect_timeout: float | None = None,
bind_address: bytes | None = None,
) -> None:
@ -164,7 +164,7 @@ class ScrapyProxyH2Agent(H2Agent):
reactor: ReactorBase,
proxy_uri: URI,
pool: H2ConnectionPool,
context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),
context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(), # noqa: B008
connect_timeout: float | None = None,
bind_address: bytes | None = None,
) -> None:

2
scrapy/core/spidermw.py
View File

@ -259,7 +259,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
# being available immediately which doesn't work when it's a wrapped coroutine.
# It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed.
@inlineCallbacks
def _process_spider_output(
def _process_spider_output( # noqa: PLR0912
self,
response: Response,
result: Iterable[_T] | AsyncIterator[_T],

10
scrapy/http/request/__init__.py
View File

@ -200,7 +200,7 @@ class Request(object_ref):
#: .. seealso:: :ref:`topics-request-response-ref-errbacks`
self.errback: Callable[[Failure], Any] | None = errback
self._cookies: CookiesT | None = cookies if cookies else None
self._cookies: CookiesT | None = cookies or None
self._headers: Headers | None = (
Headers(headers, encoding=encoding) if headers else None
)
@ -244,7 +244,7 @@ class Request(object_ref):
@cb_kwargs.setter
def cb_kwargs(self, value: dict[str, Any] | None) -> None:
self._cb_kwargs = value if value else None
self._cb_kwargs = value or None
@property
def meta(self) -> dict[str, Any]:
@ -254,7 +254,7 @@ class Request(object_ref):
@meta.setter
def meta(self, value: dict[str, Any] | None) -> None:
self._meta = value if value else None
self._meta = value or None
@property
def url(self) -> str:
@ -292,7 +292,7 @@ class Request(object_ref):
@flags.setter
def flags(self, value: list[str] | None) -> None:
self._flags = value if value else None
self._flags = value or None
@property
def cookies(self) -> CookiesT:
@ -302,7 +302,7 @@ class Request(object_ref):
@cookies.setter
def cookies(self, value: CookiesT | None) -> None:
self._cookies = value if value else None
self._cookies = value or None
@property
def headers(self) -> Headers:

2
scrapy/linkextractors/lxmlhtml.py
View File

@ -110,7 +110,7 @@ class LxmlParserLinkExtractor:
) -> list[Link]:
links: list[Link] = []
# hacky way to get the underlying lxml parsed document
for el, attr, attr_val in self._iter_links(selector.root):
for el, _, attr_val in self._iter_links(selector.root):
# pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
try:
if self.strip:

2
scrapy/responsetypes.py
View File

@ -54,7 +54,7 @@ class ResponseTypes:
return Response
if mimetype in self.classes:
return self.classes[mimetype]
basetype = f"{mimetype.split('/')[0]}/*"
basetype = f"{mimetype.split('/', maxsplit=1)[0]}/*"
return self.classes.get(basetype, Response)
def from_content_type(

1
scrapy/shell.py
View File

@ -73,7 +73,6 @@ class Shell:
else:
self.populate_vars()
if self.code:
# pylint: disable-next=eval-used
print(eval(self.code, globals(), self.vars)) # noqa: S307
else:
# Detect interactive shell setting in scrapy.cfg

2
scrapy/spidermiddlewares/referer.py
View File

@ -97,7 +97,7 @@ class ReferrerPolicy(ABC):
def potentially_trustworthy(self, url: str) -> bool:
# Note: this does not follow https://w3c.github.io/webappsec-secure-contexts/#is-url-trustworthy
parsed_url = urlparse(url)
if parsed_url.scheme in ("data",):
if parsed_url.scheme == "data":
return False
return self.tls_protected(url)

2
scrapy/utils/defer.py
View File

@ -435,7 +435,7 @@ def _maybeDeferred_coro(
"""Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""
try:
result = f(*args, **kw)
except: # noqa: E722 # pylint: disable=bare-except
except: # noqa: E722
return fail(failure.Failure(captureVars=Deferred.debug))
# when the deprecation period has ended we need to make sure the behavior

2
scrapy/utils/engine.py
View File

@ -32,7 +32,7 @@ def get_engine_status(engine: ExecutionEngine) -> list[tuple[str, Any]]:
checks: list[tuple[str, Any]] = []
for test in tests:
try:
checks += [(test, eval(test))] # noqa: S307 # pylint: disable=eval-used
checks += [(test, eval(test))] # noqa: S307
except Exception as e:
checks += [(test, f"{type(e).__name__} (exception)")]

4
scrapy/utils/log.py
View File

@ -138,7 +138,7 @@ _scrapy_root_handler: logging.Handler | None = None
def install_scrapy_root_handler(settings: Settings) -> None:
global _scrapy_root_handler # noqa: PLW0603 # pylint: disable=global-statement
global _scrapy_root_handler # noqa: PLW0603
_uninstall_scrapy_root_handler()
logging.root.setLevel(logging.NOTSET)
@ -147,7 +147,7 @@ def install_scrapy_root_handler(settings: Settings) -> None:
def _uninstall_scrapy_root_handler() -> None:
global _scrapy_root_handler # noqa: PLW0603 # pylint: disable=global-statement
global _scrapy_root_handler # noqa: PLW0603
if (
_scrapy_root_handler is not None

2
scrapy/utils/reactor.py
View File

@ -26,7 +26,7 @@ _T = TypeVar("_T")
_P = ParamSpec("_P")
def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # pylint: disable=inconsistent-return-statements # noqa: RET503
def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # noqa: RET503
"""Like reactor.listenTCP but tries different ports in a range."""
from twisted.internet import reactor

6
scrapy/utils/signal.py
View File

@ -50,7 +50,7 @@ def send_catch_log(
result: TypingAny
try:
response = robustApply(
receiver, signal=signal, sender=sender, *arguments, **named
receiver, *arguments, signal=signal, sender=sender, **named
)
if isinstance(response, Deferred):
logger.error(
@ -118,9 +118,9 @@ def _send_catch_log_deferred(
robustApply,
True,
receiver,
*arguments,
signal=signal,
sender=sender,
*arguments,
**named,
)
d.addErrback(logerror, receiver)
@ -190,7 +190,7 @@ async def _send_catch_log_asyncio(
try:
result = await ensure_awaitable(
robustApply(
receiver, signal=signal, sender=sender, *arguments, **named
receiver, *arguments, signal=signal, sender=sender, **named
),
_warn=global_object_name(receiver),
)

2
tests/mockserver/http_resources.py
View File

@ -282,7 +282,7 @@ class LargeChunkedFileResource(resource.Resource):
from twisted.internet import reactor
def response():
for i in range(1024):
for _ in range(1024):
request.write(b"x" * 1024)
request.finish()

2
tests/spiders.py
View File

@ -389,7 +389,7 @@ class DuplicateStartSpider(MockServerSpider):
async def start(self):
for i in range(self.distinct_urls):
for j in range(self.dupe_factor):
for _ in range(self.dupe_factor):
url = self.mockserver.url(f"/echo?headers=1&body=test{i}")
yield Request(url, dont_filter=self.dont_filter)

2
tests/test_downloadermiddleware_retry.py
View File

@ -250,7 +250,7 @@ class TestMaxRetryTimes:
):
middleware = middleware or self.mw
for i in range(max_retry_times):
for _ in range(max_retry_times):
req = middleware.process_exception(req, exception)
assert isinstance(req, Request)

4
tests/test_exporters.py
View File

@ -195,9 +195,7 @@ class TestPprintItemExporter(TestBaseItemExporter):
return PprintItemExporter(self.output, **kwargs)
def _check_output(self):
self._assert_expected_item(
eval(self.output.getvalue()) # pylint: disable=eval-used
)
self._assert_expected_item(eval(self.output.getvalue()))
class TestPprintItemExporterDataclass(TestPprintItemExporter):

18
tests/test_extension_periodic_log.py
View File

@ -122,8 +122,10 @@ class TestPeriodicLog:
# include multiple
check(
{"PERIODIC_LOG_DELTA": {"include": ["downloader/", "scheduler/"]}},
lambda k, v: isinstance(v, (int, float))
and ("downloader/" in k or "scheduler/" in k),
lambda k, v: (
isinstance(v, (int, float))
and ("downloader/" in k or "scheduler/" in k)
),
)
# exclude
@ -135,15 +137,19 @@ class TestPeriodicLog:
# exclude multiple
check(
{"PERIODIC_LOG_DELTA": {"exclude": ["downloader/", "scheduler/"]}},
lambda k, v: isinstance(v, (int, float))
and ("downloader/" not in k and "scheduler/" not in k),
lambda k, v: (
isinstance(v, (int, float))
and ("downloader/" not in k and "scheduler/" not in k)
),
)
# include exclude combined
check(
{"PERIODIC_LOG_DELTA": {"include": ["downloader/"], "exclude": ["bytes"]}},
lambda k, v: isinstance(v, (int, float))
and ("downloader/" in k and "bytes" not in k),
lambda k, v: (
isinstance(v, (int, float))
and ("downloader/" in k and "bytes" not in k)
),
)
@pytest.mark.requires_reactor # needs a reactor or an event loop for PeriodicLog.task

2
tests/test_item.py
View File

@ -60,7 +60,7 @@ class TestItem:
assert itemrepr == "{'name': 'John Doe', 'number': 123}"
i2 = eval(itemrepr) # pylint: disable=eval-used
i2 = eval(itemrepr)
assert i2["name"] == "John Doe"
assert i2["number"] == 123

2
tests/test_link.py
View File

@ -49,7 +49,7 @@ class TestLink:
l1 = Link(
"http://www.example.com", text="test", fragment="something", nofollow=True
)
l2 = eval(repr(l1)) # pylint: disable=eval-used
l2 = eval(repr(l1))
self._assert_same_links(l1, l2)
def test_bytes_url(self):

6
tests/test_pipeline_files.py
View File

@ -438,7 +438,7 @@ class TestFilesPipelineCustomSettings:
"""
pipe_cls = self._generate_fake_pipeline()
pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": tmp_path}))
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
for pipe_attr, _, pipe_ins_attr in self.file_cls_attr_settings_map:
custom_value = getattr(pipe, pipe_ins_attr)
assert custom_value != self.default_cls_settings[pipe_attr]
assert getattr(pipe, pipe_ins_attr) == getattr(pipe, pipe_attr)
@ -469,7 +469,7 @@ class TestFilesPipelineCustomSettings:
user_pipeline = UserDefinedFilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": tmp_path})
)
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
for pipe_attr, _, pipe_ins_attr in self.file_cls_attr_settings_map:
# Values from settings for custom pipeline should be set on pipeline instance.
custom_value = self.default_cls_settings.get(pipe_attr.upper())
assert getattr(user_pipeline, pipe_ins_attr) == custom_value
@ -541,7 +541,7 @@ class TestFilesPipelineCustomSettings:
pipeline_cls = UserPipe.from_crawler(get_crawler(None, settings))
for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
for _, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
expected_value = settings.get(settings_attr)
assert getattr(pipeline_cls, pipe_inst_attr) == expected_value

4
tests/test_pipeline_images.py
View File

@ -435,7 +435,7 @@ class TestImagesPipelineCustomSettings:
pipeline = pipeline_cls.from_crawler(
get_crawler(None, {"IMAGES_STORE": tmp_path})
)
for pipe_attr, settings_attr in self.img_cls_attribute_names:
for pipe_attr, _ in self.img_cls_attribute_names:
# Instance attribute (lowercase) must be equal to class attribute (uppercase).
attr_value = getattr(pipeline, pipe_attr.lower())
assert attr_value != self.default_pipeline_settings[pipe_attr]
@ -469,7 +469,7 @@ class TestImagesPipelineCustomSettings:
user_pipeline = UserDefinedImagePipeline.from_crawler(
get_crawler(None, {"IMAGES_STORE": tmp_path})
)
for pipe_attr, settings_attr in self.img_cls_attribute_names:
for pipe_attr, _ in self.img_cls_attribute_names:
# Values from settings for custom pipeline should be set on pipeline instance.
custom_value = self.default_pipeline_settings.get(pipe_attr.upper())
assert getattr(user_pipeline, pipe_attr.lower()) == custom_value

2
tests/test_utils_misc/test_return_with_argument_inside_generator.py
View File

@ -185,7 +185,7 @@ https://example.org
warn_on_generator_with_return_value(mock_spider, l2)
assert len(w) == 0
def test_generators_return_none_with_decorator(self, mock_spider):
def test_generators_return_none_with_decorator(self, mock_spider): # noqa: PLR0915
def decorator(func):
def inner_func():
func()

2
tests/test_utils_spider.py
View File

@ -24,7 +24,7 @@ def test_iterate_spider_output():
def test_iter_spider_classes():
import tests.test_utils_spider # noqa: PLW0406,PLC0415 # pylint: disable=import-self
import tests.test_utils_spider # noqa: PLW0406,PLC0415
it = iter_spider_classes(tests.test_utils_spider)
assert set(it) == {MySpider1, MySpider2}

2
tests/test_webclient.py
View File

@ -48,8 +48,8 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
return _makeGetterFactory(
to_bytes(url),
_clientfactory,
contextFactory=contextFactory,
*args,
contextFactory=contextFactory,
**kwargs,
).deferred