Enable the flake8-builtins rules. (#6942)

pyproject.toml

@@ -213,7 +213,6 @@ disable = [
     "keyword-arg-before-vararg",
     "pointless-statement",
     "raise-missing-from",
-    "unbalanced-tuple-unpacking",
     "unnecessary-dunder-call",
     "used-before-assignment",
 ]
@@ -234,10 +233,16 @@ filterwarnings = [
 
 [tool.ruff.lint]
 extend-select = [
+    # flake8-builtins
+    "A",
+    # flake8-async
+    "ASYNC",
     # flake8-bugbear
     "B",
     # flake8-comprehensions
     "C4",
+    # flake8-commas
+    "COM",
     # pydocstyle
     "D",
     # flake8-future-annotations
@@ -296,6 +301,8 @@ extend-select = [
 ignore = [
     # Ones we want to ignore
 
+    # Trailing comma missing
+    "COM812",
     # Missing docstring in public module
     "D100",
     # Missing docstring in public class

scrapy/commands/shell.py

@@ -56,7 +56,7 @@ class Command(ScrapyCommand):
             help="do not handle HTTP 3xx status codes and print response as-is",
         )
 
-    def update_vars(self, vars: dict[str, Any]) -> None:
+    def update_vars(self, vars: dict[str, Any]) -> None:  # noqa: A002
         """You can use this function to update the Scrapy objects that will be
         available in the shell
         """

scrapy/core/downloader/handlers/http11.py

@@ -14,7 +14,7 @@ from urllib.parse import urldefrag, urlparse
 from twisted.internet import ssl
 from twisted.internet.defer import CancelledError, Deferred, succeed
 from twisted.internet.endpoints import TCP4ClientEndpoint
-from twisted.internet.error import TimeoutError
+from twisted.internet.error import TimeoutError as TxTimeoutError
 from twisted.internet.protocol import Factory, Protocol, connectionDone
 from twisted.python.failure import Failure
 from twisted.web.client import (
@@ -452,7 +452,7 @@ class ScrapyAgent:
         if self._txresponse:
             self._txresponse._transport.stopProducing()
 
-        raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
+        raise TxTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
 
     def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:
         request.meta["download_latency"] = time() - start_time

scrapy/core/downloader/handlers/http2.py

@@ -4,7 +4,7 @@ from time import time
 from typing import TYPE_CHECKING
 from urllib.parse import urldefrag
 
-from twisted.internet.error import TimeoutError
+from twisted.internet.error import TimeoutError as TxTimeoutError
 from twisted.web.client import URI
 
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
@@ -127,4 +127,4 @@ class ScrapyH2Agent:
             return response
 
         url = urldefrag(request.url)[0]
-        raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
+        raise TxTimeoutError(f"Getting {url} took longer than {timeout} seconds.")

scrapy/core/http2/protocol.py

@@ -21,7 +21,7 @@ from h2.events import (
     WindowUpdated,
 )
 from h2.exceptions import FrameTooLargeError, H2Error
-from twisted.internet.error import TimeoutError
+from twisted.internet.error import TimeoutError as TxTimeoutError
 from twisted.internet.interfaces import (
     IAddress,
     IHandshakeListener,
@@ -322,7 +322,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         self._write_to_transport()
 
         self._lose_connection_with_error(
-            [TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")]
+            [TxTimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")]
         )
 
     def connectionLost(self, reason: Failure = connectionDone) -> None:

scrapy/downloadermiddlewares/httpcache.py

@@ -8,11 +8,11 @@ from twisted.internet.error import (
     ConnectError,
     ConnectionDone,
     ConnectionLost,
-    ConnectionRefusedError,
     DNSLookupError,
     TCPTimedOutError,
-    TimeoutError,
 )
+from twisted.internet.error import ConnectionRefusedError as TxConnectionRefusedError
+from twisted.internet.error import TimeoutError as TxTimeoutError
 from twisted.web.client import ResponseFailed
 
 from scrapy import signals
@@ -34,9 +34,9 @@ if TYPE_CHECKING:
 class HttpCacheMiddleware:
     DOWNLOAD_EXCEPTIONS = (
         defer.TimeoutError,
-        TimeoutError,
+        TxTimeoutError,
         DNSLookupError,
-        ConnectionRefusedError,
+        TxConnectionRefusedError,
         ConnectionDone,
         ConnectError,
         ConnectionLost,

scrapy/extensions/feedexport.py

@@ -376,11 +376,11 @@ class FeedSlot:
         self,
         storage: FeedStorageProtocol,
         uri: str,
-        format: str,
+        format: str,  # noqa: A002
         store_empty: bool,
         batch_id: int,
         uri_template: str,
-        filter: ItemFilter,
+        filter: ItemFilter,  # noqa: A002
         feed_options: dict[str, Any],
         spider: Spider,
         exporters: dict[str, type[BaseItemExporter]],
@@ -422,7 +422,7 @@ class FeedSlot:
                 )
             self.exporter = self._get_exporter(
                 file=self.file,
-                format=self.feed_options["format"],
+                format_=self.feed_options["format"],
                 fields_to_export=self.feed_options["fields"],
                 encoding=self.feed_options["encoding"],
                 indent=self.feed_options["indent"],
@@ -436,10 +436,10 @@ class FeedSlot:
             self._exporting = True
 
     def _get_exporter(
-        self, file: IO[bytes], format: str, *args: Any, **kwargs: Any
+        self, file: IO[bytes], format_: str, *args: Any, **kwargs: Any
     ) -> BaseItemExporter:
         return build_from_crawler(
-            self.exporters[format], self.crawler, file, *args, **kwargs
+            self.exporters[format_], self.crawler, file, *args, **kwargs
         )
 
     def finish_exporting(self) -> None:
@@ -670,10 +670,10 @@ class FeedExporter:
                 d[k] = load_object(v)
         return d
 
-    def _exporter_supported(self, format: str) -> bool:
+    def _exporter_supported(self, format_: str) -> bool:
-        if format in self.exporters:
+        if format_ in self.exporters:
             return True
-        logger.error("Unknown feed format: %(format)s", {"format": format})
+        logger.error("Unknown feed format: %(format)s", {"format": format_})
         return False
 
     def _settings_are_valid(self) -> bool:

scrapy/extensions/postprocessing.py

@@ -92,14 +92,14 @@ class LZMAPlugin:
         self.file = file
         self.feed_options = feed_options
 
-        format = self.feed_options.get("lzma_format")
+        format_ = self.feed_options.get("lzma_format")
         check = self.feed_options.get("lzma_check", -1)
         preset = self.feed_options.get("lzma_preset")
         filters = self.feed_options.get("lzma_filters")
         self.lzmafile = LZMAFile(
             filename=self.file,
             mode="wb",
-            format=format,
+            format=format_,
             check=check,
             preset=preset,
             filters=filters,

scrapy/selector/unified.py

@@ -75,7 +75,7 @@ class Selector(_ParselSelector, object_ref):
         self,
         response: TextResponse | None = None,
         text: str | None = None,
-        type: str | None = None,
+        type: str | None = None,  # noqa: A002
         root: Any | None = _NOT_SET,
         **kwargs: Any,
     ):

scrapy/utils/asyncio.py

@@ -69,7 +69,7 @@ def is_asyncio_available() -> bool:
 async def _parallel_asyncio(
     iterable: Iterable[_T] | AsyncIterator[_T],
     count: int,
-    callable: Callable[Concatenate[_T, _P], Coroutine[Any, Any, None]],
+    callable_: Callable[Concatenate[_T, _P], Coroutine[Any, Any, None]],
     *args: _P.args,
     **kwargs: _P.kwargs,
 ) -> None:
@@ -89,7 +89,7 @@ async def _parallel_asyncio(
             if item is None:
                 break
             try:
-                await callable(item, *args, **kwargs)
+                await callable_(item, *args, **kwargs)
             finally:
                 queue.task_done()
 

scrapy/utils/benchserver.py

@@ -27,8 +27,8 @@ class Root(Resource):
         return b""
 
 
-def _getarg(request, name: bytes, default: Any = None, type=str):
+def _getarg(request, name: bytes, default: Any = None, type_=str):
-    return type(request.args[name][0]) if name in request.args else default
+    return type_(request.args[name][0]) if name in request.args else default
 
 
 if __name__ == "__main__":

scrapy/utils/defer.py

@@ -157,7 +157,7 @@ def mustbe_deferred(
 def parallel(
     iterable: Iterable[_T],
     count: int,
-    callable: Callable[Concatenate[_T, _P], _T2],
+    callable: Callable[Concatenate[_T, _P], _T2],  # noqa: A002
     *args: _P.args,
     **named: _P.kwargs,
 ) -> Deferred[list[tuple[bool, Iterator[_T2]]]]:
@@ -220,12 +220,12 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
     def __init__(
         self,
         aiterable: AsyncIterator[_T],
-        callable: Callable[Concatenate[_T, _P], Deferred[Any] | None],
+        callable_: Callable[Concatenate[_T, _P], Deferred[Any] | None],
         *callable_args: _P.args,
         **callable_kwargs: _P.kwargs,
     ):
         self.aiterator: AsyncIterator[_T] = aiterable.__aiter__()
-        self.callable: Callable[Concatenate[_T, _P], Deferred[Any] | None] = callable
+        self.callable: Callable[Concatenate[_T, _P], Deferred[Any] | None] = callable_
         self.callable_args: tuple[Any, ...] = callable_args
         self.callable_kwargs: dict[str, Any] = callable_kwargs
         self.finished: bool = False
@@ -278,7 +278,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
 def parallel_async(
     async_iterable: AsyncIterator[_T],
     count: int,
-    callable: Callable[Concatenate[_T, _P], Deferred[Any] | None],
+    callable: Callable[Concatenate[_T, _P], Deferred[Any] | None],  # noqa: A002
     *args: _P.args,
     **named: _P.kwargs,
 ) -> Deferred[list[tuple[bool, Iterator[Deferred[Any]]]]]:
@@ -295,7 +295,7 @@ def parallel_async(
 
 def process_chain(
     callbacks: Iterable[Callable[Concatenate[_T, _P], _T]],
-    input: _T,
+    input: _T,  # noqa: A002
     *a: _P.args,
     **kw: _P.kwargs,
 ) -> Deferred[_T]:
@@ -310,7 +310,7 @@ def process_chain(
 def process_chain_both(
     callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
     errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]],
-    input: Any,
+    input: Any,  # noqa: A002
     *a: _P.args,
     **kw: _P.kwargs,
 ) -> Deferred:
@@ -334,7 +334,7 @@ def process_chain_both(
 
 def process_parallel(
     callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]],
-    input: _T,
+    input: _T,  # noqa: A002
     *a: _P.args,
     **kw: _P.kwargs,
 ) -> Deferred[list[_T2]]:

scrapy/utils/misc.py

@@ -241,7 +241,7 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
 _generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
 
 
-def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:
+def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:  # noqa: A002
     """
     Returns True if a callable is a generator function which includes a
     'return' statement with a value different than None, False otherwise
@@ -279,7 +279,8 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:
 
 
 def warn_on_generator_with_return_value(
-    spider: Spider, callable: Callable[..., Any]
+    spider: Spider,
+    callable: Callable[..., Any],  # noqa: A002
 ) -> None:
     """
     Logs a warning if a callable is a generator function and includes

tests/mocks/dummydbm.py

@@ -17,7 +17,7 @@ error = KeyError
 _DATABASES: defaultdict[Any, DummyDB] = defaultdict(DummyDB)
 
 
-def open(file, flag="r", mode=0o666):
+def open(file, flag="r", mode=0o666):  # noqa: A001
     """Open or create a dummy database compatible.
 
     Arguments ``flag`` and ``mode`` are ignored.

tests/mockserver.py

@@ -29,11 +29,11 @@ if TYPE_CHECKING:
     from twisted.internet.protocol import ServerFactory
 
 
-def getarg(request, name, default=None, type=None):
+def getarg(request, name, default=None, type_=None):
     if name in request.args:
         value = request.args[name][0]
-        if type is not None:
+        if type_ is not None:
-            value = type(value)
+            value = type_(value)
         return value
     return default
 
@@ -129,11 +129,11 @@ class LeafResource(resource.Resource):
 
 class Follow(LeafResource):
     def render(self, request):
-        total = getarg(request, b"total", 100, type=int)
+        total = getarg(request, b"total", 100, type_=int)
-        show = getarg(request, b"show", 1, type=int)
+        show = getarg(request, b"show", 1, type_=int)
         order = getarg(request, b"order", b"desc")
-        maxlatency = getarg(request, b"maxlatency", 0, type=float)
+        maxlatency = getarg(request, b"maxlatency", 0, type_=float)
-        n = getarg(request, b"n", total, type=int)
+        n = getarg(request, b"n", total, type_=int)
         if order == b"rand":
             nlist = [random.randint(1, total) for _ in range(show)]
         else:  # order == "desc"
@@ -157,8 +157,8 @@ class Follow(LeafResource):
 
 class Delay(LeafResource):
     def render_GET(self, request):
-        n = getarg(request, b"n", 1, type=float)
+        n = getarg(request, b"n", 1, type_=float)
-        b = getarg(request, b"b", 1, type=int)
+        b = getarg(request, b"b", 1, type_=int)
         if b:
             # send headers now and delay body
             request.write("")
@@ -172,7 +172,7 @@ class Delay(LeafResource):
 
 class Status(LeafResource):
     def render_GET(self, request):
-        n = getarg(request, b"n", 200, type=int)
+        n = getarg(request, b"n", 200, type_=int)
         request.setResponseCode(n)
         return b""
 
@@ -229,7 +229,7 @@ class Partial(LeafResource):
 
 class Drop(Partial):
     def _delayedRender(self, request):
-        abort = getarg(request, b"abort", 0, type=int)
+        abort = getarg(request, b"abort", 0, type_=int)
         request.write(b"this connection will be dropped\n")
         tr = request.channel.transport
         try:

tests/test_downloadermiddleware_retry.py

@@ -10,6 +10,8 @@ from twisted.internet.error import (
     DNSLookupError,
     TCPTimedOutError,
 )
+from twisted.internet.error import ConnectionRefusedError as TxConnectionRefusedError
+from twisted.internet.error import TimeoutError as TxTimeoutError
 from twisted.web.client import ResponseFailed
 
 from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request
@@ -91,12 +93,12 @@ class TestRetry:
             ConnectError,
             ConnectionDone,
             ConnectionLost,
-            ConnectionRefusedError,
+            TxConnectionRefusedError,
             defer.TimeoutError,
             DNSLookupError,
             ResponseFailed,
             TCPTimedOutError,
-            TimeoutError,
+            TxTimeoutError,
         ]
 
         for exc in exceptions:

tests/test_dupefilters.py

@@ -16,12 +16,12 @@ from scrapy.utils.test import get_crawler
 from tests.spiders import SimpleSpider
 
 
-def _get_dupefilter(*, crawler=None, settings=None, open=True):
+def _get_dupefilter(*, crawler=None, settings=None, open_=True):
     if crawler is None:
         crawler = get_crawler(settings_dict=settings)
     scheduler = Scheduler.from_crawler(crawler)
     dupefilter = scheduler.df
-    if open:
+    if open_:
         dupefilter.open()
     return dupefilter
 
@@ -77,7 +77,7 @@ class TestRFPDupeFilter:
 
         path = tempfile.mkdtemp()
         try:
-            df = _get_dupefilter(settings={"JOBDIR": path}, open=False)
+            df = _get_dupefilter(settings={"JOBDIR": path}, open_=False)
             try:
                 df.open()
                 assert not df.request_seen(r1)
@@ -85,7 +85,7 @@ class TestRFPDupeFilter:
             finally:
                 df.close("finished")
 
-            df2 = _get_dupefilter(settings={"JOBDIR": path}, open=False)
+            df2 = _get_dupefilter(settings={"JOBDIR": path}, open_=False)
             assert df != df2
             try:
                 df2.open()

tests/test_http2_client_protocol.py

@@ -20,7 +20,7 @@ from twisted.internet.defer import (
     inlineCallbacks,
 )
 from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
-from twisted.internet.error import TimeoutError
+from twisted.internet.error import TimeoutError as TxTimeoutError
 from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS
 from twisted.trial.unittest import TestCase
 from twisted.web.client import URI, ResponseFailed
@@ -632,7 +632,7 @@ class TestHttps2ClientProtocol(TestCase):
         for err in exc_info.value.reasons:
             from scrapy.core.http2.protocol import H2ClientProtocol  # noqa: PLC0415
 
-            if isinstance(err, TimeoutError):
+            if isinstance(err, TxTimeoutError):
                 assert (
                     f"Connection was IDLE for more than {H2ClientProtocol.IDLE_TIMEOUT}s"
                     in str(err)

tests/test_pipeline_images.py

@@ -502,8 +502,8 @@ class TestImagesPipelineCustomSettings:
             assert getattr(pipeline_cls, pipe_attr.lower()) == expected_value
 
 
-def _create_image(format, *a, **kw):
+def _create_image(format_, *a, **kw):
     buf = io.BytesIO()
-    Image.new(*a, **kw).save(buf, format)
+    Image.new(*a, **kw).save(buf, format_)
     buf.seek(0)
     return Image.open(buf), buf

tests/test_pqueues.py

@@ -159,7 +159,7 @@ class TestDownloaderAwarePriorityQueue:
 
 
 @pytest.mark.parametrize(
-    ("input", "output"),
+    ("input_", "output"),
     [
         # By default, start requests are FIFO, other requests are LIFO.
         ([{}, {}], [2, 1]),
@@ -172,7 +172,7 @@ class TestDownloaderAwarePriorityQueue:
         ([{"start": True}, {}], [2, 1]),
     ],
 )
-def test_pop_order(input, output):
+def test_pop_order(input_, output):
     def make_url(index):
         return f"https://toscrape.com/{index}"
 
@@ -187,7 +187,7 @@ def test_pop_order(input, output):
         )
 
     input_requests = [
-        make_request(index, data) for index, data in enumerate(input, start=1)
+        make_request(index, data) for index, data in enumerate(input_, start=1)
     ]
     expected_output_urls = [make_url(index) for index in output]
 

tests/test_utils_datatypes.py

@@ -353,26 +353,26 @@ class TestLocalWeakReferencedCache:
         assert len(cache) == 0
 
     def test_cache_without_limit(self):
-        max = 10**4
+        maximum = 10**4
         cache = LocalWeakReferencedCache()
         refs = []
-        for x in range(max):
+        for x in range(maximum):
             refs.append(Request(f"https://example.org/{x}"))
             cache[refs[-1]] = x
-        assert len(cache) == max
+        assert len(cache) == maximum
         for i, r in enumerate(refs):
             assert r in cache
             assert cache[r] == i
         del r  # delete reference to the last object in the list  # pylint: disable=undefined-loop-variable
 
         # delete half of the objects, make sure that is reflected in the cache
-        for _ in range(max // 2):
+        for _ in range(maximum // 2):
             refs.pop()
 
         # PyPy takes longer to collect dead references
         garbage_collect()
 
-        assert len(cache) == max // 2
+        assert len(cache) == maximum // 2
         for i, r in enumerate(refs):
             assert r in cache
             assert cache[r] == i

tests/test_utils_display.py

@@ -77,10 +77,10 @@ def test_pformat_no_pygments(isatty):
 
     real_import = builtins.__import__
 
-    def mock_import(name, globals, locals, fromlist, level):
+    def mock_import(name, globals_, locals_, fromlist, level):
         if "pygments" in name:
             raise ImportError
-        return real_import(name, globals, locals, fromlist, level)
+        return real_import(name, globals_, locals_, fromlist, level)
 
     builtins.__import__ = mock_import
     assert pformat(value) == plain_string

tests/test_utils_iterators.py

@@ -231,11 +231,11 @@ class TestXmliterBase(ABC):
             "<products><product>one</product><product>two</product></products>"
         )
 
-        iter = self.xmliter(body, "product")
+        my_iter = self.xmliter(body, "product")
-        next(iter)
+        next(my_iter)
-        next(iter)
+        next(my_iter)
         with pytest.raises(StopIteration):
-            next(iter)
+            next(my_iter)
 
     def test_xmliter_objtype_exception(self):
         i = self.xmliter(42, "product")
@@ -473,13 +473,13 @@ class TestUtilsCsv:
         body = get_testdata("feeds", "feed-sample3.csv")
 
         response = TextResponse(url="http://example.com/", body=body)
-        iter = csviter(response)
+        my_iter = csviter(response)
-        next(iter)
+        next(my_iter)
-        next(iter)
+        next(my_iter)
-        next(iter)
+        next(my_iter)
-        next(iter)
+        next(my_iter)
         with pytest.raises(StopIteration):
-            next(iter)
+            next(my_iter)
 
     def test_csviter_encoding(self):
         body1 = get_testdata("feeds", "feed-sample4.csv")

tests/test_utils_serialize.py

@@ -30,7 +30,7 @@ class TestJsonEncoder:
         dt_set = {dt}
         dt_sets = [dts]
 
-        for input, output in [
+        for input_, output in [
             ("foo", "foo"),
             (d, ds),
             (t, ts),
@@ -40,7 +40,7 @@ class TestJsonEncoder:
             (s, ss),
             (dt_set, dt_sets),
         ]:
-            assert encoder.encode(input) == json.dumps(output, sort_keys=True)
+            assert encoder.encode(input_) == json.dumps(output, sort_keys=True)
 
     def test_encode_deferred(self, encoder: ScrapyJSONEncoder) -> None:
         assert "Deferred" in encoder.encode(defer.Deferred())