chore: simplify some code, get rid of nested fns where it's makes sence + pylint (#7401)

* chore: simplify some code, get rid of nested fns where it's makes sence

reasoning: https://wemake-python-styleguide.readthedocs.io/en/latest/pages/usage/violations/best_practices.html#wemake_python_styleguide.violations.best_practices.NestedFunctionViolation

* chore: address some pylint rules

* remove `pylint: disable=deprecated-class` from `reactor.py`

* fix typo

* fix another typo :)

* revert backward incompatible arg renames

---------

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>

extras/qps-bench-server.py

@@ -18,7 +18,7 @@ class Root(Resource):
         self.tail.clear()
         self.start = self.lastmark = self.lasttime = time()
 
-    def getChild(self, request, name):
+    def getChild(self, path, request):
         return self
 
     def render(self, request):

pyproject.toml

@@ -185,6 +185,7 @@ jobs = 1          # >1 hides results
 extension-pkg-allow-list=[
     "lxml",
 ]
+load-plugins = ["pylint_per_file_ignores"]
 
 [tool.pylint."MESSAGES CONTROL"]
 enable = [
@@ -244,15 +245,13 @@ disable = [
     "unused-import",
 
     # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
-    "abstract-method",
     "arguments-differ",
-    "arguments-renamed",
-    "dangerous-default-value",
     "keyword-arg-before-vararg",
-    "pointless-statement",
+]
-    "raise-missing-from",
+# requires `pylint_per_file_ignores` plugin
-    "unnecessary-dunder-call",
+per-file-ignores = [
-    "used-before-assignment",
+    # Extended list of ones that we may want to address, only for tests
+    "./tests/*:abstract-method,arguments-renamed,dangerous-default-value,pointless-statement,raise-missing-from,unnecessary-dunder-call,used-before-assignment",
 ]
 
 [tool.pytest.ini_options]

scrapy/contracts/__init__.py

@@ -125,7 +125,7 @@ class ContractsManager:
     def from_spider(self, spider: Spider, results: TestResult) -> list[Request | None]:
         requests: list[Request | None] = []
         for method in self.tested_methods_from_spidercls(type(spider)):
-            bound_method = spider.__getattribute__(method)
+            bound_method = getattr(spider, method)
             try:
                 requests.append(self.from_method(bound_method, results))
             except Exception:

scrapy/core/downloader/handlers/http11.py

@@ -6,6 +6,7 @@ import ipaddress
 import logging
 import re
 from contextlib import suppress
+from functools import partial
 from io import BytesIO
 from time import monotonic
 from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast
@@ -547,11 +548,7 @@ class ScrapyAgent:
                 get_warnsize_msg(expected_size, warnsize, request, expected=True)
             )
 
-        def _cancel(_: Any) -> None:
+        d: Deferred[_ResultT] = Deferred(partial(self._cancel, txresponse=txresponse))
-            # Abort connection immediately.
-            txresponse._transport._producer.abortConnection()
-
-        d: Deferred[_ResultT] = Deferred(_cancel)
         txresponse.deliverBody(
             _ResponseReader(
                 finished=d,
@@ -570,6 +567,11 @@ class ScrapyAgent:
 
         return d
 
+    @staticmethod
+    def _cancel(_: Any, txresponse: TxResponse) -> None:
+        # Abort connection immediately.
+        txresponse._transport._producer.abortConnection()
+
     def _cb_bodydone(self, result: _ResultT, url: str) -> Response:
         headers = self._headers_from_twisted_response(result["txresponse"])
         try:
@@ -667,17 +669,17 @@ class _ResponseReader(Protocol):
             assert hostname is not None
             _log_ssl_conn_debug_info(hostname, connection)
 
-    def dataReceived(self, bodyBytes: bytes) -> None:
+    def dataReceived(self, data: bytes) -> None:
         # This maybe called several times after cancel was called with buffered data.
         if self._finished.called:
             return
 
         assert self.transport
-        self._bodybuf.write(bodyBytes)
+        self._bodybuf.write(data)
-        self._bytes_received += len(bodyBytes)
+        self._bytes_received += len(data)
 
         if stop_download := check_stop_download(
-            signals.bytes_received, self._crawler, self._request, data=bodyBytes
+            signals.bytes_received, self._crawler, self._request, data=data
         ):
             self.transport.stopProducing()
             self.transport.loseConnection()

scrapy/core/downloader/middleware.py

@@ -75,87 +75,82 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         download_func: Callable[[Request], Coroutine[Any, Any, Response]],
         request: Request,
     ) -> Response | Request:
-        async def process_request(request: Request) -> Response | Request:
-            for method in self.methods["process_request"]:
-                method = cast("Callable", method)
-                if method in self._mw_methods_requiring_spider:
-                    response = await ensure_awaitable(
-                        method(request=request, spider=self._spider),
-                        _warn=global_object_name(method),
-                    )
-                else:
-                    response = await ensure_awaitable(
-                        method(request=request), _warn=global_object_name(method)
-                    )
-                if response is not None and not isinstance(
-                    response, (Response, Request)
-                ):
-                    raise _InvalidOutput(
-                        f"Middleware {method.__qualname__} must return None, Response or "
-                        f"Request, got {response.__class__.__name__}"
-                    )
-                if response:
-                    return response
-            return await download_func(request)
-
-        async def process_response(response: Response | Request) -> Response | Request:
-            if response is None:
-                raise TypeError("Received None in process_response")
-            if isinstance(response, Request):
-                return response
-
-            for method in self.methods["process_response"]:
-                method = cast("Callable", method)
-                if method in self._mw_methods_requiring_spider:
-                    response = await ensure_awaitable(
-                        method(request=request, response=response, spider=self._spider),
-                        _warn=global_object_name(method),
-                    )
-                else:
-                    response = await ensure_awaitable(
-                        method(request=request, response=response),
-                        _warn=global_object_name(method),
-                    )
-                if not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput(
-                        f"Middleware {method.__qualname__} must return Response or Request, "
-                        f"got {type(response)}"
-                    )
-                if isinstance(response, Request):
-                    return response
-            return response
-
-        async def process_exception(exception: Exception) -> Response | Request:
-            for method in self.methods["process_exception"]:
-                method = cast("Callable", method)
-                if method in self._mw_methods_requiring_spider:
-                    response = await ensure_awaitable(
-                        method(
-                            request=request, exception=exception, spider=self._spider
-                        ),
-                        _warn=global_object_name(method),
-                    )
-                else:
-                    response = await ensure_awaitable(
-                        method(request=request, exception=exception),
-                        _warn=global_object_name(method),
-                    )
-                if response is not None and not isinstance(
-                    response, (Response, Request)
-                ):
-                    raise _InvalidOutput(
-                        f"Middleware {method.__qualname__} must return None, Response or "
-                        f"Request, got {type(response)}"
-                    )
-                if response:
-                    return response
-            raise exception
 
         try:
-            result: Response | Request = await process_request(request)
+            result: Response | Request = await self._process_request(
+                request, download_func
+            )
         except Exception as ex:
             await _defer_sleep_async()
             # either returns a request or response (which we pass to process_response())
             # or reraises the exception
-            result = await process_exception(ex)
+            result = await self._process_exception(ex, request)
-        return await process_response(result)
+        return await self._process_response(result, request)
+
+    def _handle_mw_method(self, method: Callable, **kwargs: Any) -> Any:
+        if method in self._mw_methods_requiring_spider:
+            kwargs["spider"] = self._spider
+
+        return method(**kwargs)
+
+    async def _process_request(
+        self,
+        request: Request,
+        download_func: Callable[[Request], Coroutine[Any, Any, Response]],
+    ) -> Response | Request:
+        for method in self.methods["process_request"]:
+            method = cast("Callable", method)
+            response = await ensure_awaitable(
+                self._handle_mw_method(method, request=request),
+                _warn=global_object_name(method),
+            )
+            if response is not None and not isinstance(response, (Response, Request)):
+                raise _InvalidOutput(
+                    f"Middleware {method.__qualname__} must return None, Response or "
+                    f"Request, got {response.__class__.__name__}"
+                )
+            if response:
+                return response
+        return await download_func(request)
+
+    async def _process_response(
+        self, response: Response | Request, request: Request
+    ) -> Response | Request:
+        if response is None:
+            raise TypeError("Received None in process_response")
+        if isinstance(response, Request):
+            return response
+
+        for method in self.methods["process_response"]:
+            method = cast("Callable", method)
+            response = await ensure_awaitable(
+                self._handle_mw_method(method, request=request, response=response),
+                _warn=global_object_name(method),
+            )
+
+            if not isinstance(response, (Response, Request)):
+                raise _InvalidOutput(
+                    f"Middleware {method.__qualname__} must return Response or Request, "
+                    f"got {type(response)}"
+                )
+            if isinstance(response, Request):
+                return response
+        return response
+
+    async def _process_exception(
+        self, exception: Exception, request: Request | Response
+    ) -> Response | Request:
+        for method in self.methods["process_exception"]:
+            method = cast("Callable", method)
+            response = await ensure_awaitable(
+                self._handle_mw_method(method, request=request, exception=exception),
+                _warn=global_object_name(method),
+            )
+            if response is not None and not isinstance(response, (Response, Request)):
+                raise _InvalidOutput(
+                    f"Middleware {method.__qualname__} must return None, Response or "
+                    f"Request, got {type(response)}"
+                )
+            if response:
+                return response
+        raise exception

scrapy/core/engine.py

@@ -11,6 +11,7 @@ import asyncio
 import contextlib
 import logging
 import warnings
+from functools import partial
 from time import time
 from traceback import format_exc
 from typing import TYPE_CHECKING, Any
@@ -273,7 +274,7 @@ class ExecutionEngine:
         """
         assert self._start is not None
         try:
-            item_or_request = await self._start.__anext__()
+            item_or_request = await anext(self._start)
         except StopAsyncIteration:
             self._start = None
         except Exception as exception:
@@ -352,6 +353,10 @@ class ExecutionEngine:
             or self.scraper.slot.needs_backout()
         )
 
+    def _remove_request(self, _: Any, request: Request) -> None:
+        assert self._slot
+        self._slot.remove_request(request)
+
     def _start_scheduled_request(self) -> bool:
         assert self._slot is not None  # typing
         assert self.spider is not None  # typing
@@ -371,11 +376,7 @@ class ExecutionEngine:
             )
         )
 
-        def _remove_request(_: Any) -> None:
+        d2: Deferred[None] = d.addBoth(partial(self._remove_request, request=request))
-            assert self._slot
-            self._slot.remove_request(request)
-
-        d2: Deferred[None] = d.addBoth(_remove_request)
         d2.addErrback(
             lambda f: logger.info(
                 "Error while removing request from slot",
@@ -612,30 +613,33 @@ class ExecutionEngine:
             "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider}
         )
 
-        def log_failure(msg: str) -> None:
-            logger.error(msg, exc_info=True, extra={"spider": spider})  # noqa: LOG014
-
         try:
             await self._slot.close()
         except Exception:
-            log_failure("Slot close failure")
+            logger.error("Slot close failure", exc_info=True, extra={"spider": spider})
 
         try:
             self.downloader.close()
         except Exception:
-            log_failure("Downloader close failure")
+            logger.error(
+                "Downloader close failure", exc_info=True, extra={"spider": spider}
+            )
 
         try:
             await self.scraper.close_spider_async()
         except Exception:
-            log_failure("Scraper close failure")
+            logger.error(
+                "Scraper close failure", exc_info=True, extra={"spider": spider}
+            )
 
         if hasattr(self._slot.scheduler, "close"):
             try:
                 if (d := self._slot.scheduler.close(reason)) is not None:
                     await maybe_deferred_to_future(d)
             except Exception:
-                log_failure("Scheduler close failure")
+                logger.error(
+                    "Scheduler close failure", exc_info=True, extra={"spider": spider}
+                )
 
         try:
             await self.signals.send_catch_log_async(
@@ -644,7 +648,11 @@ class ExecutionEngine:
                 reason=reason,
             )
         except Exception:
-            log_failure("Error while sending spider_close signal")
+            logger.error(
+                "Error while sending spider_close signal",
+                exc_info=True,
+                extra={"spider": spider},
+            )
 
         assert self.crawler.stats
         try:
@@ -661,7 +669,7 @@ class ExecutionEngine:
             else:
                 self.crawler.stats.close_spider(reason=reason)
         except Exception:
-            log_failure("Stats close failure")
+            logger.error("Stats close failure")
 
         logger.info(
             "Spider closed (%(reason)s)",
@@ -675,4 +683,4 @@ class ExecutionEngine:
         try:
             await ensure_awaitable(self._spider_closed_callback(spider))
         except Exception:
-            log_failure("Error running spider_closed_callback")
+            logger.error("Error running spider_closed_callback")

scrapy/core/http2/stream.py

@@ -155,16 +155,16 @@ class Stream:
             "status": None,
         }
 
-        def _cancel(_: Any) -> None:
+        self._deferred_response: Deferred[Response] = Deferred(self._cancel)
-            # Close this stream as gracefully as possible
-            # If the associated request is initiated we reset this stream
-            # else we directly call close() method
-            if self.metadata["request_sent"]:
-                self.reset_stream(StreamCloseReason.CANCELLED)
-            else:
-                self.close(StreamCloseReason.CANCELLED)
 
-        self._deferred_response: Deferred[Response] = Deferred(_cancel)
+    def _cancel(self, _: Any) -> None:
+        # Close this stream as gracefully as possible
+        # If the associated request is initiated we reset this stream
+        # else we directly call close() method
+        if self.metadata["request_sent"]:
+            self.reset_stream(StreamCloseReason.CANCELLED)
+        else:
+            self.close(StreamCloseReason.CANCELLED)
 
     def __repr__(self) -> str:
         return f"Stream(id={self.stream_id!r})"

scrapy/core/spidermw.py

@@ -112,40 +112,59 @@ class SpiderMiddlewareManager(MiddlewareManager):
         exception_processor_index: int,
         recover_to: MutableChain[_T] | MutableAsyncChain[_T],
     ) -> Iterable[_T] | AsyncIterator[_T]:
-        def process_sync(iterable: Iterable[_T]) -> Iterable[_T]:
-            try:
-                yield from iterable
-            except Exception as ex:
-                exception_result = cast(
-                    "Failure | MutableChain[_T]",
-                    self._process_spider_exception(
-                        response, ex, exception_processor_index
-                    ),
-                )
-                if isinstance(exception_result, Failure):
-                    raise
-                assert isinstance(recover_to, MutableChain)
-                recover_to.extend(exception_result)
-
-        async def process_async(iterable: AsyncIterator[_T]) -> AsyncIterator[_T]:
-            try:
-                async for r in iterable:
-                    yield r
-            except Exception as ex:
-                exception_result = cast(
-                    "Failure | MutableAsyncChain[_T]",
-                    self._process_spider_exception(
-                        response, ex, exception_processor_index
-                    ),
-                )
-                if isinstance(exception_result, Failure):
-                    raise
-                assert isinstance(recover_to, MutableAsyncChain)
-                recover_to.extend(exception_result)
 
         if isinstance(iterable, AsyncIterator):
-            return process_async(iterable)
+            return self._process_async(
-        return process_sync(iterable)
+                response,
+                iterable,
+                exception_processor_index,
+                cast("MutableAsyncChain[_T]", recover_to),
+            )
+        return self._process_sync(
+            response,
+            iterable,
+            exception_processor_index,
+            cast("MutableChain[_T]", recover_to),
+        )
+
+    def _process_sync(
+        self,
+        response: Response,
+        iterable: Iterable[_T],
+        exception_processor_index: int,
+        recover_to: MutableChain[_T],
+    ) -> Iterable[_T]:
+        try:
+            yield from iterable
+        except Exception as ex:
+            exception_result = cast(
+                "Failure | MutableChain[_T]",
+                self._process_spider_exception(response, ex, exception_processor_index),
+            )
+            if isinstance(exception_result, Failure):
+                raise
+            assert isinstance(recover_to, MutableChain)
+            recover_to.extend(exception_result)
+
+    async def _process_async(
+        self,
+        response: Response,
+        iterable: AsyncIterator[_T],
+        exception_processor_index: int,
+        recover_to: MutableAsyncChain[_T],
+    ) -> AsyncIterator[_T]:
+        try:
+            async for r in iterable:
+                yield r
+        except Exception as ex:
+            exception_result = cast(
+                "Failure | MutableAsyncChain[_T]",
+                self._process_spider_exception(response, ex, exception_processor_index),
+            )
+            if isinstance(exception_result, Failure):
+                raise
+            assert isinstance(recover_to, MutableAsyncChain)
+            recover_to.extend(exception_result)
 
     def _process_spider_exception(
         self,
@@ -350,25 +369,14 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 "scrape_response_async() called on a SpiderMiddlewareManager"
                 " instance created without a crawler."
             )
-
-        async def process_callback_output(
-            result: Iterable[_T] | AsyncIterator[_T],
-        ) -> MutableChain[_T] | MutableAsyncChain[_T]:
-            return await self._process_callback_output(response, result)
-
-        def process_spider_exception(
-            exception: Exception,
-        ) -> MutableChain[_T] | MutableAsyncChain[_T]:
-            return self._process_spider_exception(response, exception)
-
         try:
             it: Iterable[_T] | AsyncIterator[_T] = await self._process_spider_input(
                 scrape_func, response, request
             )
-            return await process_callback_output(it)
+            return await self._process_callback_output(response, it)
         except Exception as ex:
             await _defer_sleep_async()
-            return process_spider_exception(ex)
+            return self._process_spider_exception(response, ex)
 
     async def process_start(
         self, spider: Spider | None = None

scrapy/crawler.py

@@ -7,6 +7,7 @@ import pprint
 import signal
 import warnings
 from abc import ABC, abstractmethod
+from functools import partial
 from typing import TYPE_CHECKING, Any, TypeVar
 
 from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
@@ -568,28 +569,30 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
         crawler = self.create_crawler(crawler_or_spidercls)
         return self._crawl(crawler, *args, **kwargs)
 
+    async def _crawl_and_track(
+        self, crawler: Crawler, *args: Any, **kwargs: Any
+    ) -> None:
+        try:
+            await crawler.crawl_async(*args, **kwargs)
+        except Exception:
+            self.bootstrap_failed = True
+            raise  # re-raise so asyncio still logs it to stderr naturally
+
+    def _done(self, task: asyncio.Task[None], crawler: Crawler) -> None:
+        self._active.discard(task)
+        self.crawlers.discard(crawler)
+        self.bootstrap_failed |= not getattr(crawler, "spider", None)
+
     def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]:
         # At this point the asyncio loop has been installed either by the user
         # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()).
         loop = asyncio.get_event_loop()
         self.crawlers.add(crawler)
 
-        async def _crawl_and_track() -> None:
+        task = loop.create_task(self._crawl_and_track(crawler, *args, **kwargs))
-            try:
-                await crawler.crawl_async(*args, **kwargs)
-            except Exception:
-                self.bootstrap_failed = True
-                raise  # re-raise so asyncio still logs it to stderr naturally
-
-        task = loop.create_task(_crawl_and_track())
         self._active.add(task)
+        task.add_done_callback(partial(self._done, crawler=crawler))
 
-        def _done(_: asyncio.Task[None]) -> None:
-            self.crawlers.discard(crawler)
-            self._active.discard(task)
-            self.bootstrap_failed |= not getattr(crawler, "spider", None)
-
-        task.add_done_callback(_done)
         return task
 
     async def stop(self) -> None:
@@ -1007,14 +1010,15 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
         if (loop := self._reactorless_loop) is None:
             return
 
-        def _create_shutdown_task() -> None:
+        loop.call_soon_threadsafe(self._create_shutdown_task)
-            coro = self._shutdown_graceful_reactorless()
-            try:
-                loop.create_task(coro)
-            except RuntimeError:
-                coro.close()
 
-        loop.call_soon_threadsafe(_create_shutdown_task)
+    def _create_shutdown_task(self) -> None:
+        assert self._reactorless_loop
+        coro = self._shutdown_graceful_reactorless()
+        try:
+            self._reactorless_loop.create_task(coro)
+        except RuntimeError:
+            coro.close()
 
     async def _shutdown_graceful_reactorless(self) -> None:
         await self.stop()

scrapy/extensions/feedexport.py

@@ -534,13 +534,15 @@ class FeedExporter:
         # Send FEED_EXPORTER_CLOSED signal
         await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed)
 
+    @staticmethod
+    def _get_file(slot_: FeedSlot) -> IO[bytes]:
+        assert slot_.file
+        if isinstance(slot_.file, PostProcessingManager):
+            slot_.file.close()
+            return slot_.file.file
+        return slot_.file
+
     async def _close_slot(self, slot: FeedSlot, spider: Spider) -> None:
-        def get_file(slot_: FeedSlot) -> IO[bytes]:
-            assert slot_.file
-            if isinstance(slot_.file, PostProcessingManager):
-                slot_.file.close()
-                return slot_.file.file
-            return slot_.file
 
         if slot.itemcount:
             # Normal case
@@ -557,7 +559,7 @@ class FeedExporter:
         slot_type = type(slot.storage).__name__
         assert self.crawler.stats
         try:
-            await ensure_awaitable(slot.storage.store(get_file(slot)))
+            await ensure_awaitable(slot.storage.store(self._get_file(slot)))
         except Exception:
             logger.error(
                 "Error storing %s",

scrapy/pipelines/__init__.py

@@ -37,16 +37,16 @@ class ItemPipelineManager(MiddlewareManager):
             settings.get_component_priority_dict_with_base("ITEM_PIPELINES")
         )
 
-    def _add_middleware(self, pipe: Any) -> None:
+    def _add_middleware(self, mw: Any) -> None:
-        if hasattr(pipe, "open_spider"):
+        if hasattr(mw, "open_spider"):
-            self.methods["open_spider"].append(pipe.open_spider)
+            self.methods["open_spider"].append(mw.open_spider)
-            self._check_mw_method_spider_arg(pipe.open_spider)
+            self._check_mw_method_spider_arg(mw.open_spider)
-        if hasattr(pipe, "close_spider"):
+        if hasattr(mw, "close_spider"):
-            self.methods["close_spider"].appendleft(pipe.close_spider)
+            self.methods["close_spider"].appendleft(mw.close_spider)
-            self._check_mw_method_spider_arg(pipe.close_spider)
+            self._check_mw_method_spider_arg(mw.close_spider)
-        if hasattr(pipe, "process_item"):
+        if hasattr(mw, "process_item"):
-            self.methods["process_item"].append(pipe.process_item)
+            self.methods["process_item"].append(mw.process_item)
-            self._check_mw_method_spider_arg(pipe.process_item)
+            self._check_mw_method_spider_arg(mw.process_item)
 
     def process_item(self, item: Any, spider: Spider) -> Deferred[Any]:
         warnings.warn(
@@ -62,32 +62,44 @@ class ItemPipelineManager(MiddlewareManager):
             "process_item", item, add_spider=True, warn_deferred=True
         )
 
+    def _get_dfd(
+        self,
+        method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
+    ) -> Deferred[None]:
+        if method in self._mw_methods_requiring_spider:
+            return _maybeDeferred_coro(method, True, self._spider)
+        return _maybeDeferred_coro(method, True)
+
+    @staticmethod
+    def _eb(failure: Failure) -> Failure:
+        assert isinstance(failure.value, FirstError)
+        return failure.value.subFailure
+
     def _process_parallel_dfd(self, methodname: str) -> Deferred[list[None]]:
         methods = cast(
             "Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]",
             self.methods[methodname],
         )
 
-        def get_dfd(
+        dfds = [self._get_dfd(m) for m in methods]
-            method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
-        ) -> Deferred[None]:
-            if method in self._mw_methods_requiring_spider:
-                return _maybeDeferred_coro(method, True, self._spider)
-            return _maybeDeferred_coro(method, True)
-
-        dfds = [get_dfd(m) for m in methods]
         d: Deferred[list[tuple[bool, None]]] = DeferredList(
             dfds, fireOnOneErrback=True, consumeErrors=True
         )
         d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r])
 
-        def eb(failure: Failure) -> Failure:
+        d2.addErrback(self._eb)
-            assert isinstance(failure.value, FirstError)
-            return failure.value.subFailure
-
-        d2.addErrback(eb)
         return d2
 
+    def get_awaitable(
+        self,
+        method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
+    ) -> Awaitable[None]:
+        if method in self._mw_methods_requiring_spider:
+            result = method(self._spider)
+        else:
+            result = method()
+        return ensure_awaitable(result, _warn=global_object_name(method))
+
     async def _process_parallel_asyncio(self, methodname: str) -> list[None]:
         methods = cast(
             "Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]",
@@ -96,16 +108,7 @@ class ItemPipelineManager(MiddlewareManager):
         if not methods:
             return []
 
-        def get_awaitable(
+        awaitables = [self.get_awaitable(m) for m in methods]
-            method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
-        ) -> Awaitable[None]:
-            if method in self._mw_methods_requiring_spider:
-                result = method(self._spider)
-            else:
-                result = method()
-            return ensure_awaitable(result, _warn=global_object_name(method))
-
-        awaitables = [get_awaitable(m) for m in methods]
         await asyncio.gather(*awaitables)
         return [None for _ in methods]
 

scrapy/pipelines/files.py

@@ -186,16 +186,18 @@ class S3FilesStore:
             raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
         self.bucket, self.prefix = uri[5:].split("/", 1)
 
+    @staticmethod
+    def _onsuccess(boto_key: dict[str, Any]) -> StatInfo:
+        checksum = boto_key["ETag"].strip('"')
+        last_modified = boto_key["LastModified"]
+        modified_stamp = time.mktime(last_modified.timetuple())
+        return {"checksum": checksum, "last_modified": modified_stamp}
+
     def stat_file(
         self, path: str, info: MediaPipeline.SpiderInfo
     ) -> Deferred[StatInfo]:
-        def _onsuccess(boto_key: dict[str, Any]) -> StatInfo:
-            checksum = boto_key["ETag"].strip('"')
-            last_modified = boto_key["LastModified"]
-            modified_stamp = time.mktime(last_modified.timetuple())
-            return {"checksum": checksum, "last_modified": modified_stamp}
 
-        return self._get_boto_key(path).addCallback(_onsuccess)
+        return self._get_boto_key(path).addCallback(self._onsuccess)
 
     def _get_boto_key(self, path: str) -> Deferred[dict[str, Any]]:
         key_name = f"{self.prefix}{path}"
@@ -308,20 +310,22 @@ class GCSFilesStore:
                 {"bucket": bucket},
             )
 
+    @staticmethod
+    def _onsuccess(blob: Any) -> StatInfo:
+        if blob:
+            checksum = base64.b64decode(blob.md5_hash).hex()
+            last_modified = time.mktime(blob.updated.timetuple())
+            return {"checksum": checksum, "last_modified": last_modified}
+        return {}
+
     def stat_file(
         self, path: str, info: MediaPipeline.SpiderInfo
     ) -> Deferred[StatInfo]:
-        def _onsuccess(blob: Any) -> StatInfo:
-            if blob:
-                checksum = base64.b64decode(blob.md5_hash).hex()
-                last_modified = time.mktime(blob.updated.timetuple())
-                return {"checksum": checksum, "last_modified": last_modified}
-            return {}
 
         blob_path = self._get_blob_path(path)
         return deferred_from_coro(
             run_in_thread(self.bucket.get_blob, blob_path)
-        ).addCallback(_onsuccess)
+        ).addCallback(self._onsuccess)
 
     def _get_content_type(self, headers: dict[str, str] | None) -> str:
         if headers and "Content-Type" in headers:
@@ -395,26 +399,26 @@ class FTPFilesStore:
             )
         )
 
+    def _stat_file(self, path: str) -> StatInfo:
+        try:
+            with FTP() as ftp:
+                ftp.connect(self.host, self.port)
+                ftp.login(self.username, self.password)
+                if self.USE_ACTIVE_MODE:
+                    ftp.set_pasv(False)
+                file_path = f"{self.basedir}/{path}"
+                last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())
+                m = hashlib.md5()  # noqa: S324
+                ftp.retrbinary(f"RETR {file_path}", m.update)
+            return {"last_modified": last_modified, "checksum": m.hexdigest()}
+        # The file doesn't exist
+        except Exception:
+            return {}
+
     def stat_file(
         self, path: str, info: MediaPipeline.SpiderInfo
     ) -> Deferred[StatInfo]:
-        def _stat_file(path: str) -> StatInfo:
+        return deferred_from_coro(run_in_thread(self._stat_file, path))
-            try:
-                with FTP() as ftp:
-                    ftp.connect(self.host, self.port)
-                    ftp.login(self.username, self.password)
-                    if self.USE_ACTIVE_MODE:
-                        ftp.set_pasv(False)
-                    file_path = f"{self.basedir}/{path}"
-                    last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())
-                    m = hashlib.md5()  # noqa: S324
-                    ftp.retrbinary(f"RETR {file_path}", m.update)
-                return {"last_modified": last_modified, "checksum": m.hexdigest()}
-            # The file doesn't exist
-            except Exception:
-                return {}
-
-        return deferred_from_coro(run_in_thread(_stat_file, path))
 
 
 class FilesPipeline(MediaPipeline):
@@ -534,43 +538,51 @@ class FilesPipeline(MediaPipeline):
         store_cls = self.STORE_SCHEMES[scheme]
         return store_cls(uri)
 
+    def _onsuccess(
+        self,
+        result: StatInfo,
+        request: Request,
+        info: MediaPipeline.SpiderInfo,
+        path: str,
+    ) -> FileInfo | None:
+        if not result:
+            return None  # returning None force download
+
+        last_modified = result.get("last_modified", None)
+        if not last_modified:
+            return None  # returning None force download
+
+        age_seconds = time.time() - last_modified
+        age_days = age_seconds / 60 / 60 / 24
+        if age_days > self.expires:
+            return None  # returning None force download
+
+        referer = referer_str(request)
+        logger.debug(
+            "File (uptodate): Downloaded %(medianame)s from %(request)s "
+            "referred in <%(referer)s>",
+            {"medianame": self.MEDIA_NAME, "request": request, "referer": referer},
+            extra={"spider": info.spider},
+        )
+        self.inc_stats("uptodate")
+
+        checksum = result.get("checksum", None)
+        return {
+            "url": request.url,
+            "path": path,
+            "checksum": checksum,
+            "status": "uptodate",
+        }
+
     def media_to_download(
         self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None
     ) -> Deferred[FileInfo | None] | None:
-        def _onsuccess(result: StatInfo) -> FileInfo | None:
-            if not result:
-                return None  # returning None force download
-
-            last_modified = result.get("last_modified", None)
-            if not last_modified:
-                return None  # returning None force download
-
-            age_seconds = time.time() - last_modified
-            age_days = age_seconds / 60 / 60 / 24
-            if age_days > self.expires:
-                return None  # returning None force download
-
-            referer = referer_str(request)
-            logger.debug(
-                "File (uptodate): Downloaded %(medianame)s from %(request)s "
-                "referred in <%(referer)s>",
-                {"medianame": self.MEDIA_NAME, "request": request, "referer": referer},
-                extra={"spider": info.spider},
-            )
-            self.inc_stats("uptodate")
-
-            checksum = result.get("checksum", None)
-            return {
-                "url": request.url,
-                "path": path,
-                "checksum": checksum,
-                "status": "uptodate",
-            }
-
         path = self.file_path(request, info=info, item=item)
         # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
         dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info)  # type: ignore[call-overload]
-        dfd2: Deferred[FileInfo | None] = dfd.addCallback(_onsuccess)
+        dfd2: Deferred[FileInfo | None] = dfd.addCallback(
+            functools.partial(self._onsuccess, request=request, info=info, path=path)
+        )
         dfd2.addErrback(lambda _: None)
         dfd2.addErrback(
             lambda f: logger.error(

scrapy/settings/__init__.py

@@ -143,7 +143,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
             raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).")
         self.set(name, [v for v in value if v != item], self.getpriority(name) or 0)
 
-    def get(self, name: _SettingsKey, default: Any = None) -> Any:
+    def get(self, name: _SettingsKey, default: Any = None) -> Any:  # pylint: disable=arguments-renamed
         """
         Get a setting value without affecting its original type.
 
@@ -510,7 +510,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
         component_priority_dict[cls] = priority
         self.set(name, component_priority_dict, self.getpriority(name) or 0)
 
-    def setdefault(
+    def setdefault(  # pylint: disable=arguments-renamed
         self,
         name: _SettingsKey,
         default: Any = None,
@@ -691,14 +691,14 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
         else:
             p.text(pformat(self.copy_to_dict()))
 
-    def pop(self, name: _SettingsKey, default: Any = __default) -> Any:
+    def pop(self, name: _SettingsKey, default: Any = __default) -> Any:  # pylint: disable=arguments-renamed
         try:
             value = self.attributes[name].value
         except KeyError:
             if default is self.__default:
                 raise
             return default
-        self.__delitem__(name)
+        del self[name]
         return value
 
 

scrapy/utils/benchserver.py

@@ -9,7 +9,7 @@ from twisted.web.server import Request, Site
 class Root(Resource):
     isLeaf = True
 
-    def getChild(self, name: str, request: Request) -> Resource:
+    def getChild(self, path: str, request: Request) -> Resource:
         return self
 
     def render(self, request: Request) -> bytes:

scrapy/utils/defer.py

@@ -262,7 +262,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
     def _call_anext(self) -> None:
         # This starts waiting for the next result from aiterator.
         # If aiterator is exhausted, _errback will be called.
-        self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
+        self.anext_deferred = deferred_from_coro(anext(self.aiterator))
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
     def __next__(self) -> Deferred[Any]:
@@ -370,10 +370,10 @@ async def aiter_errback(
     """Wrap an async iterable calling an errback if an error is caught while
     iterating it. Similar to :func:`scrapy.utils.defer.iter_errback`.
     """
-    it = aiterable.__aiter__()
+    it = aiter(aiterable)
     while True:
         try:
-            yield await it.__anext__()
+            yield await anext(it)
         except StopAsyncIteration:
             break
         except Exception:

scrapy/utils/misc.py

@@ -255,6 +255,11 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
 _generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
 
 
+def _returns_none(return_node: ast.Return) -> bool:
+    value = return_node.value
+    return value is None or (isinstance(value, ast.Constant) and value.value is None)
+
+
 def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:  # noqa: A002
     """
     Returns True if a callable is a generator function which includes a
@@ -263,12 +268,6 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:  # noq
     if callable in _generator_callbacks_cache:
         return bool(_generator_callbacks_cache[callable])
 
-    def returns_none(return_node: ast.Return) -> bool:
-        value = return_node.value
-        return value is None or (
-            isinstance(value, ast.Constant) and value.value is None
-        )
-
     if inspect.isgeneratorfunction(callable):
         func = callable
         while isinstance(func, partial):
@@ -284,7 +283,7 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:  # noq
 
         tree = ast.parse(code)
         for node in walk_callable(tree):
-            if isinstance(node, ast.Return) and not returns_none(node):
+            if isinstance(node, ast.Return) and not _returns_none(node):
                 _generator_callbacks_cache[callable] = True
                 return bool(_generator_callbacks_cache[callable])
 

scrapy/utils/python.py

@@ -99,6 +99,16 @@ def to_bytes(
     return text.encode(encoding, errors)
 
 
+def _chunk_iter(text: str, chunk_size: int) -> Iterable[tuple[str, int]]:
+    offset = len(text)
+    while True:
+        offset -= chunk_size * 1024
+        if offset <= 0:
+            break
+        yield (text[offset:], offset)
+    yield (text, 0)
+
+
 def re_rsearch(
     pattern: str | Pattern[str], text: str, chunk_size: int = 1024
 ) -> tuple[int, int] | None:
@@ -115,19 +125,10 @@ def re_rsearch(
     the start position of the match, and the ending (regarding the entire text).
     """
 
-    def _chunk_iter() -> Iterable[tuple[str, int]]:
-        offset = len(text)
-        while True:
-            offset -= chunk_size * 1024
-            if offset <= 0:
-                break
-            yield (text[offset:], offset)
-        yield (text, 0)
-
     if isinstance(pattern, str):
         pattern = re.compile(pattern)
 
-    for chunk, offset in _chunk_iter():
+    for chunk, offset in _chunk_iter(text, chunk_size):
         matches = list(pattern.finditer(chunk))
         if matches:
             start, end = matches[-1].span()

tests/mockserver/http.py

@@ -85,7 +85,7 @@ class Root(resource.Resource):
         self.putChild(b"response-headers", ResponseHeadersResource())
         self.putChild(b"set-cookie", SetCookie())
 
-    def getChild(self, name, request):
+    def getChild(self, path, request):
         return self
 
     def render(self, request):

tests/mockserver/simple_https.py

@@ -13,7 +13,7 @@ class Root(resource.Resource):
         resource.Resource.__init__(self)
         self.putChild(b"file", Data(b"0123456789", "text/plain"))
 
-    def getChild(self, name, request):
+    def getChild(self, path, request):
         return self
 
 

tox.ini

@@ -89,8 +89,9 @@ basepython = python3
 deps =
     {[testenv:extra-deps]deps}
     pylint==4.0.2
+    pylint-per-file-ignores # https://github.com/pylint-dev/pylint/issues/3767#issuecomment-1319916278
 commands =
-    pylint conftest.py docs extras scrapy tests
+    pylint {posargs:conftest.py docs extras scrapy tests}
 
 [testenv:twinecheck]
 basepython = python3