RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Enable the flake8-builtins rules. (#6942)

This commit is contained in:
Andrey Rakhmatullin 2025-07-05 21:43:18 +05:00 committed by GitHub
parent df342eee6e
commit 4279f2837c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
24 changed files with 90 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pyproject.toml
View File

@ -213,7 +213,6 @@ disable = [
"keyword-arg-before-vararg",
"pointless-statement",
"raise-missing-from",
"unbalanced-tuple-unpacking",
"unnecessary-dunder-call",
"used-before-assignment",
]
@ -234,10 +233,16 @@ filterwarnings = [
[tool.ruff.lint]
extend-select = [
# flake8-builtins
"A",
# flake8-async
"ASYNC",
# flake8-bugbear
"B",
# flake8-comprehensions
"C4",
# flake8-commas
"COM",
# pydocstyle
"D",
# flake8-future-annotations
@ -296,6 +301,8 @@ extend-select = [
ignore = [
# Ones we want to ignore
# Trailing comma missing
"COM812",
# Missing docstring in public module
"D100",
# Missing docstring in public class

2
scrapy/commands/shell.py
View File

@ -56,7 +56,7 @@ class Command(ScrapyCommand):
help="do not handle HTTP 3xx status codes and print response as-is",
)
def update_vars(self, vars: dict[str, Any]) -> None:
def update_vars(self, vars: dict[str, Any]) -> None: # noqa: A002
"""You can use this function to update the Scrapy objects that will be
available in the shell
"""

4
scrapy/core/downloader/handlers/http11.py
View File

@ -14,7 +14,7 @@ from urllib.parse import urldefrag, urlparse
from twisted.internet import ssl
from twisted.internet.defer import CancelledError, Deferred, succeed
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.internet.error import TimeoutError
from twisted.internet.error import TimeoutError as TxTimeoutError
from twisted.internet.protocol import Factory, Protocol, connectionDone
from twisted.python.failure import Failure
from twisted.web.client import (
@ -452,7 +452,7 @@ class ScrapyAgent:
if self._txresponse:
self._txresponse._transport.stopProducing()
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
raise TxTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:
request.meta["download_latency"] = time() - start_time

4
scrapy/core/downloader/handlers/http2.py
View File

@ -4,7 +4,7 @@ from time import time
from typing import TYPE_CHECKING
from urllib.parse import urldefrag
from twisted.internet.error import TimeoutError
from twisted.internet.error import TimeoutError as TxTimeoutError
from twisted.web.client import URI
from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
@ -127,4 +127,4 @@ class ScrapyH2Agent:
return response
url = urldefrag(request.url)[0]
raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
raise TxTimeoutError(f"Getting {url} took longer than {timeout} seconds.")

4
scrapy/core/http2/protocol.py
View File

@ -21,7 +21,7 @@ from h2.events import (
WindowUpdated,
)
from h2.exceptions import FrameTooLargeError, H2Error
from twisted.internet.error import TimeoutError
from twisted.internet.error import TimeoutError as TxTimeoutError
from twisted.internet.interfaces import (
IAddress,
IHandshakeListener,
@ -322,7 +322,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
self._write_to_transport()
self._lose_connection_with_error(
[TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")]
[TxTimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")]
)
def connectionLost(self, reason: Failure = connectionDone) -> None:

8
scrapy/downloadermiddlewares/httpcache.py
View File

@ -8,11 +8,11 @@ from twisted.internet.error import (
ConnectError,
ConnectionDone,
ConnectionLost,
ConnectionRefusedError,
DNSLookupError,
TCPTimedOutError,
TimeoutError,
)
from twisted.internet.error import ConnectionRefusedError as TxConnectionRefusedError
from twisted.internet.error import TimeoutError as TxTimeoutError
from twisted.web.client import ResponseFailed
from scrapy import signals
@ -34,9 +34,9 @@ if TYPE_CHECKING:
class HttpCacheMiddleware:
DOWNLOAD_EXCEPTIONS = (
defer.TimeoutError,
TimeoutError,
TxTimeoutError,
DNSLookupError,
ConnectionRefusedError,
TxConnectionRefusedError,
ConnectionDone,
ConnectError,
ConnectionLost,

16
scrapy/extensions/feedexport.py
View File

@ -376,11 +376,11 @@ class FeedSlot:
self,
storage: FeedStorageProtocol,
uri: str,
format: str,
format: str, # noqa: A002
store_empty: bool,
batch_id: int,
uri_template: str,
filter: ItemFilter,
filter: ItemFilter, # noqa: A002
feed_options: dict[str, Any],
spider: Spider,
exporters: dict[str, type[BaseItemExporter]],
@ -422,7 +422,7 @@ class FeedSlot:
)
self.exporter = self._get_exporter(
file=self.file,
format=self.feed_options["format"],
format_=self.feed_options["format"],
fields_to_export=self.feed_options["fields"],
encoding=self.feed_options["encoding"],
indent=self.feed_options["indent"],
@ -436,10 +436,10 @@ class FeedSlot:
self._exporting = True
def _get_exporter(
self, file: IO[bytes], format: str, *args: Any, **kwargs: Any
self, file: IO[bytes], format_: str, *args: Any, **kwargs: Any
) -> BaseItemExporter:
return build_from_crawler(
self.exporters[format], self.crawler, file, *args, **kwargs
self.exporters[format_], self.crawler, file, *args, **kwargs
)
def finish_exporting(self) -> None:
@ -670,10 +670,10 @@ class FeedExporter:
d[k] = load_object(v)
return d
def _exporter_supported(self, format: str) -> bool:
if format in self.exporters:
def _exporter_supported(self, format_: str) -> bool:
if format_ in self.exporters:
return True
logger.error("Unknown feed format: %(format)s", {"format": format})
logger.error("Unknown feed format: %(format)s", {"format": format_})
return False
def _settings_are_valid(self) -> bool:

4
scrapy/extensions/postprocessing.py
View File

@ -92,14 +92,14 @@ class LZMAPlugin:
self.file = file
self.feed_options = feed_options
format = self.feed_options.get("lzma_format")
format_ = self.feed_options.get("lzma_format")
check = self.feed_options.get("lzma_check", -1)
preset = self.feed_options.get("lzma_preset")
filters = self.feed_options.get("lzma_filters")
self.lzmafile = LZMAFile(
filename=self.file,
mode="wb",
format=format,
format=format_,
check=check,
preset=preset,
filters=filters,

2
scrapy/selector/unified.py
View File

@ -75,7 +75,7 @@ class Selector(_ParselSelector, object_ref):
self,
response: TextResponse | None = None,
text: str | None = None,
type: str | None = None,
type: str | None = None, # noqa: A002
root: Any | None = _NOT_SET,
**kwargs: Any,
):

4
scrapy/utils/asyncio.py
View File

@ -69,7 +69,7 @@ def is_asyncio_available() -> bool:
async def _parallel_asyncio(
iterable: Iterable[_T] | AsyncIterator[_T],
count: int,
callable: Callable[Concatenate[_T, _P], Coroutine[Any, Any, None]],
callable_: Callable[Concatenate[_T, _P], Coroutine[Any, Any, None]],
*args: _P.args,
**kwargs: _P.kwargs,
) -> None:
@ -89,7 +89,7 @@ async def _parallel_asyncio(
if item is None:
break
try:
await callable(item, *args, **kwargs)
await callable_(item, *args, **kwargs)
finally:
queue.task_done()

4
scrapy/utils/benchserver.py
View File

@ -27,8 +27,8 @@ class Root(Resource):
return b""
def _getarg(request, name: bytes, default: Any = None, type=str):
return type(request.args[name][0]) if name in request.args else default
def _getarg(request, name: bytes, default: Any = None, type_=str):
return type_(request.args[name][0]) if name in request.args else default
if __name__ == "__main__":

14
scrapy/utils/defer.py
View File

@ -157,7 +157,7 @@ def mustbe_deferred(
def parallel(
iterable: Iterable[_T],
count: int,
callable: Callable[Concatenate[_T, _P], _T2],
callable: Callable[Concatenate[_T, _P], _T2], # noqa: A002
*args: _P.args,
**named: _P.kwargs,
) -> Deferred[list[tuple[bool, Iterator[_T2]]]]:
@ -220,12 +220,12 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
def __init__(
self,
aiterable: AsyncIterator[_T],
callable: Callable[Concatenate[_T, _P], Deferred[Any] | None],
callable_: Callable[Concatenate[_T, _P], Deferred[Any] | None],
*callable_args: _P.args,
**callable_kwargs: _P.kwargs,
):
self.aiterator: AsyncIterator[_T] = aiterable.__aiter__()
self.callable: Callable[Concatenate[_T, _P], Deferred[Any] | None] = callable
self.callable: Callable[Concatenate[_T, _P], Deferred[Any] | None] = callable_
self.callable_args: tuple[Any, ...] = callable_args
self.callable_kwargs: dict[str, Any] = callable_kwargs
self.finished: bool = False
@ -278,7 +278,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
def parallel_async(
async_iterable: AsyncIterator[_T],
count: int,
callable: Callable[Concatenate[_T, _P], Deferred[Any] | None],
callable: Callable[Concatenate[_T, _P], Deferred[Any] | None], # noqa: A002
*args: _P.args,
**named: _P.kwargs,
) -> Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]]:
@ -295,7 +295,7 @@ def parallel_async(
def process_chain(
callbacks: Iterable[Callable[Concatenate[_T, _P], _T]],
input: _T,
input: _T, # noqa: A002
*a: _P.args,
**kw: _P.kwargs,
) -> Deferred[_T]:
@ -310,7 +310,7 @@ def process_chain(
def process_chain_both(
callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]],
input: Any,
input: Any, # noqa: A002
*a: _P.args,
**kw: _P.kwargs,
) -> Deferred:
@ -334,7 +334,7 @@ def process_chain_both(
def process_parallel(
callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]],
input: _T,
input: _T, # noqa: A002
*a: _P.args,
**kw: _P.kwargs,
) -> Deferred[list[_T2]]:

5
scrapy/utils/misc.py
View File

@ -241,7 +241,7 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:
def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: # noqa: A002
"""
Returns True if a callable is a generator function which includes a
'return' statement with a value different than None, False otherwise
@ -279,7 +279,8 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:
def warn_on_generator_with_return_value(
spider: Spider, callable: Callable[..., Any]
spider: Spider,
callable: Callable[..., Any], # noqa: A002
) -> None:
"""
Logs a warning if a callable is a generator function and includes

2
tests/mocks/dummydbm.py
View File

@ -17,7 +17,7 @@ error = KeyError
_DATABASES: defaultdict[Any, DummyDB] = defaultdict(DummyDB)
def open(file, flag="r", mode=0o666):
def open(file, flag="r", mode=0o666): # noqa: A001
"""Open or create a dummy database compatible.
Arguments ``flag`` and ``mode`` are ignored.

22
tests/mockserver.py
View File

@ -29,11 +29,11 @@ if TYPE_CHECKING:
from twisted.internet.protocol import ServerFactory
def getarg(request, name, default=None, type=None):
def getarg(request, name, default=None, type_=None):
if name in request.args:
value = request.args[name][0]
if type is not None:
value = type(value)
if type_ is not None:
value = type_(value)
return value
return default
@ -129,11 +129,11 @@ class LeafResource(resource.Resource):
class Follow(LeafResource):
def render(self, request):
total = getarg(request, b"total", 100, type=int)
show = getarg(request, b"show", 1, type=int)
total = getarg(request, b"total", 100, type_=int)
show = getarg(request, b"show", 1, type_=int)
order = getarg(request, b"order", b"desc")
maxlatency = getarg(request, b"maxlatency", 0, type=float)
n = getarg(request, b"n", total, type=int)
maxlatency = getarg(request, b"maxlatency", 0, type_=float)
n = getarg(request, b"n", total, type_=int)
if order == b"rand":
nlist = [random.randint(1, total) for _ in range(show)]
else: # order == "desc"
@ -157,8 +157,8 @@ class Follow(LeafResource):
class Delay(LeafResource):
def render_GET(self, request):
n = getarg(request, b"n", 1, type=float)
b = getarg(request, b"b", 1, type=int)
n = getarg(request, b"n", 1, type_=float)
b = getarg(request, b"b", 1, type_=int)
if b:
# send headers now and delay body
request.write("")
@ -172,7 +172,7 @@ class Delay(LeafResource):
class Status(LeafResource):
def render_GET(self, request):
n = getarg(request, b"n", 200, type=int)
n = getarg(request, b"n", 200, type_=int)
request.setResponseCode(n)
return b""
@ -229,7 +229,7 @@ class Partial(LeafResource):
class Drop(Partial):
def _delayedRender(self, request):
abort = getarg(request, b"abort", 0, type=int)
abort = getarg(request, b"abort", 0, type_=int)
request.write(b"this connection will be dropped\n")
tr = request.channel.transport
try:

6
tests/test_downloadermiddleware_retry.py
View File

@ -10,6 +10,8 @@ from twisted.internet.error import (
DNSLookupError,
TCPTimedOutError,
)
from twisted.internet.error import ConnectionRefusedError as TxConnectionRefusedError
from twisted.internet.error import TimeoutError as TxTimeoutError
from twisted.web.client import ResponseFailed
from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request
@ -91,12 +93,12 @@ class TestRetry:
ConnectError,
ConnectionDone,
ConnectionLost,
ConnectionRefusedError,
TxConnectionRefusedError,
defer.TimeoutError,
DNSLookupError,
ResponseFailed,
TCPTimedOutError,
TimeoutError,
TxTimeoutError,
]
for exc in exceptions:

8
tests/test_dupefilters.py
View File

@ -16,12 +16,12 @@ from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
def _get_dupefilter(*, crawler=None, settings=None, open=True):
def _get_dupefilter(*, crawler=None, settings=None, open_=True):
if crawler is None:
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
dupefilter = scheduler.df
if open:
if open_:
dupefilter.open()
return dupefilter
@ -77,7 +77,7 @@ class TestRFPDupeFilter:
path = tempfile.mkdtemp()
try:
df = _get_dupefilter(settings={"JOBDIR": path}, open=False)
df = _get_dupefilter(settings={"JOBDIR": path}, open_=False)
try:
df.open()
assert not df.request_seen(r1)
@ -85,7 +85,7 @@ class TestRFPDupeFilter:
finally:
df.close("finished")
df2 = _get_dupefilter(settings={"JOBDIR": path}, open=False)
df2 = _get_dupefilter(settings={"JOBDIR": path}, open_=False)
assert df != df2
try:
df2.open()

4
tests/test_http2_client_protocol.py
View File

@ -20,7 +20,7 @@ from twisted.internet.defer import (
inlineCallbacks,
)
from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
from twisted.internet.error import TimeoutError
from twisted.internet.error import TimeoutError as TxTimeoutError
from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS
from twisted.trial.unittest import TestCase
from twisted.web.client import URI, ResponseFailed
@ -632,7 +632,7 @@ class TestHttps2ClientProtocol(TestCase):
for err in exc_info.value.reasons:
from scrapy.core.http2.protocol import H2ClientProtocol # noqa: PLC0415
if isinstance(err, TimeoutError):
if isinstance(err, TxTimeoutError):
assert (
f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s"
in str(err)

4
tests/test_pipeline_images.py
View File

@ -502,8 +502,8 @@ class TestImagesPipelineCustomSettings:
assert getattr(pipeline_cls, pipe_attr.lower()) == expected_value
def _create_image(format, *a, **kw):
def _create_image(format_, *a, **kw):
buf = io.BytesIO()
Image.new(*a, **kw).save(buf, format)
Image.new(*a, **kw).save(buf, format_)
buf.seek(0)
return Image.open(buf), buf

6
tests/test_pqueues.py
View File

@ -159,7 +159,7 @@ class TestDownloaderAwarePriorityQueue:
@pytest.mark.parametrize(
("input", "output"),
("input_", "output"),
[
# By default, start requests are FIFO, other requests are LIFO.
([{}, {}], [2, 1]),
@ -172,7 +172,7 @@ class TestDownloaderAwarePriorityQueue:
([{"start": True}, {}], [2, 1]),
],
)
def test_pop_order(input, output):
def test_pop_order(input_, output):
def make_url(index):
return f"https://toscrape.com/{index}"
@ -187,7 +187,7 @@ def test_pop_order(input, output):
)
input_requests = [
make_request(index, data) for index, data in enumerate(input, start=1)
make_request(index, data) for index, data in enumerate(input_, start=1)
]
expected_output_urls = [make_url(index) for index in output]

10
tests/test_utils_datatypes.py
View File

@ -353,26 +353,26 @@ class TestLocalWeakReferencedCache:
assert len(cache) == 0
def test_cache_without_limit(self):
max = 10**4
maximum = 10**4
cache = LocalWeakReferencedCache()
refs = []
for x in range(max):
for x in range(maximum):
refs.append(Request(f"https://example.org/{x}"))
cache[refs[-1]] = x
assert len(cache) == max
assert len(cache) == maximum
for i, r in enumerate(refs):
assert r in cache
assert cache[r] == i
del r # delete reference to the last object in the list # pylint: disable=undefined-loop-variable
# delete half of the objects, make sure that is reflected in the cache
for _ in range(max // 2):
for _ in range(maximum // 2):
refs.pop()
# PyPy takes longer to collect dead references
garbage_collect()
assert len(cache) == max // 2
assert len(cache) == maximum // 2
for i, r in enumerate(refs):
assert r in cache
assert cache[r] == i

4
tests/test_utils_display.py
View File

@ -77,10 +77,10 @@ def test_pformat_no_pygments(isatty):
real_import = builtins.__import__
def mock_import(name, globals, locals, fromlist, level):
def mock_import(name, globals_, locals_, fromlist, level):
if "pygments" in name:
raise ImportError
return real_import(name, globals, locals, fromlist, level)
return real_import(name, globals_, locals_, fromlist, level)
builtins.__import__ = mock_import
assert pformat(value) == plain_string

20
tests/test_utils_iterators.py
View File

@ -231,11 +231,11 @@ class TestXmliterBase(ABC):
"<products><product>one</product><product>two</product></products>"
)
iter = self.xmliter(body, "product")
next(iter)
next(iter)
my_iter = self.xmliter(body, "product")
next(my_iter)
next(my_iter)
with pytest.raises(StopIteration):
next(iter)
next(my_iter)
def test_xmliter_objtype_exception(self):
i = self.xmliter(42, "product")
@ -473,13 +473,13 @@ class TestUtilsCsv:
body = get_testdata("feeds", "feed-sample3.csv")
response = TextResponse(url="http://example.com/", body=body)
iter = csviter(response)
next(iter)
next(iter)
next(iter)
next(iter)
my_iter = csviter(response)
next(my_iter)
next(my_iter)
next(my_iter)
next(my_iter)
with pytest.raises(StopIteration):
next(iter)
next(my_iter)
def test_csviter_encoding(self):
body1 = get_testdata("feeds", "feed-sample4.csv")

4
tests/test_utils_serialize.py
View File

@ -30,7 +30,7 @@ class TestJsonEncoder:
dt_set = {dt}
dt_sets = [dts]
for input, output in [
for input_, output in [
("foo", "foo"),
(d, ds),
(t, ts),
@ -40,7 +40,7 @@ class TestJsonEncoder:
(s, ss),
(dt_set, dt_sets),
]:
assert encoder.encode(input) == json.dumps(output, sort_keys=True)
assert encoder.encode(input_) == json.dumps(output, sort_keys=True)
def test_encode_deferred(self, encoder: ScrapyJSONEncoder) -> None:
assert "Deferred" in encoder.encode(defer.Deferred())