chore: simplify some code, get rid of nested fns where it's makes sence + pylint (#7401)

* chore: simplify some code, get rid of nested fns where it's makes sence

reasoning: https://wemake-python-styleguide.readthedocs.io/en/latest/pages/usage/violations/best_practices.html#wemake_python_styleguide.violations.best_practices.NestedFunctionViolation

* chore: address some pylint rules

* remove `pylint: disable=deprecated-class` from `reactor.py`

* fix typo

* fix another typo :)

* revert backward incompatible arg renames

---------

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>

extras/qps-bench-server.py

@@ -18,7 +18,7 @@ class Root(Resource):
         self.tail.clear()
         self.start = self.lastmark = self.lasttime = time()
 
-    def getChild(self, request, name):
+    def getChild(self, path, request):
         return self
 
     def render(self, request):

pyproject.toml

@@ -185,6 +185,7 @@ jobs = 1          # >1 hides results
 extension-pkg-allow-list=[
     "lxml",
 ]
+load-plugins = ["pylint_per_file_ignores"]
 
 [tool.pylint."MESSAGES CONTROL"]
 enable = [
@@ -244,15 +245,13 @@ disable = [
     "unused-import",
 
     # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
-    "abstract-method",
     "arguments-differ",
-    "arguments-renamed",
-    "dangerous-default-value",
     "keyword-arg-before-vararg",
-    "pointless-statement",
+]
-    "raise-missing-from",
+# requires `pylint_per_file_ignores` plugin
-    "unnecessary-dunder-call",
+per-file-ignores = [
-    "used-before-assignment",
+    # Extended list of ones that we may want to address, only for tests
+    "./tests/*:abstract-method,arguments-renamed,dangerous-default-value,pointless-statement,raise-missing-from,unnecessary-dunder-call,used-before-assignment",
 ]
 
 [tool.pytest.ini_options]

scrapy/contracts/__init__.py

@@ -125,7 +125,7 @@ class ContractsManager:
     def from_spider(self, spider: Spider, results: TestResult) -> list[Request | None]:
         requests: list[Request | None] = []
         for method in self.tested_methods_from_spidercls(type(spider)):
-            bound_method = spider.__getattribute__(method)
+            bound_method = getattr(spider, method)
             try:
                 requests.append(self.from_method(bound_method, results))
             except Exception:

scrapy/core/downloader/handlers/http11.py

@@ -6,6 +6,7 @@ import ipaddress
 import logging
 import re
 from contextlib import suppress
+from functools import partial
 from io import BytesIO
 from time import monotonic
 from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast
@@ -547,11 +548,7 @@ class ScrapyAgent:
                 get_warnsize_msg(expected_size, warnsize, request, expected=True)
             )
 
-        def _cancel(_: Any) -> None:
+        d: Deferred[_ResultT] = Deferred(partial(self._cancel, txresponse=txresponse))
-            # Abort connection immediately.
-            txresponse._transport._producer.abortConnection()
-
-        d: Deferred[_ResultT] = Deferred(_cancel)
         txresponse.deliverBody(
             _ResponseReader(
                 finished=d,
@@ -570,6 +567,11 @@ class ScrapyAgent:
 
         return d
 
+    @staticmethod
+    def _cancel(_: Any, txresponse: TxResponse) -> None:
+        # Abort connection immediately.
+        txresponse._transport._producer.abortConnection()
+
     def _cb_bodydone(self, result: _ResultT, url: str) -> Response:
         headers = self._headers_from_twisted_response(result["txresponse"])
         try:
@@ -667,17 +669,17 @@ class _ResponseReader(Protocol):
             assert hostname is not None
             _log_ssl_conn_debug_info(hostname, connection)
 
-    def dataReceived(self, bodyBytes: bytes) -> None:
+    def dataReceived(self, data: bytes) -> None:
         # This maybe called several times after cancel was called with buffered data.
         if self._finished.called:
             return
 
         assert self.transport
-        self._bodybuf.write(bodyBytes)
+        self._bodybuf.write(data)
-        self._bytes_received += len(bodyBytes)
+        self._bytes_received += len(data)
 
         if stop_download := check_stop_download(
-            signals.bytes_received, self._crawler, self._request, data=bodyBytes
+            signals.bytes_received, self._crawler, self._request, data=data
         ):
             self.transport.stopProducing()
             self.transport.loseConnection()

scrapy/core/downloader/middleware.py

@@ -75,21 +75,36 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         download_func: Callable[[Request], Coroutine[Any, Any, Response]],
         request: Request,
     ) -> Response | Request:
-        async def process_request(request: Request) -> Response | Request:
+
+        try:
+            result: Response | Request = await self._process_request(
+                request, download_func
+            )
+        except Exception as ex:
+            await _defer_sleep_async()
+            # either returns a request or response (which we pass to process_response())
+            # or reraises the exception
+            result = await self._process_exception(ex, request)
+        return await self._process_response(result, request)
+
+    def _handle_mw_method(self, method: Callable, **kwargs: Any) -> Any:
+        if method in self._mw_methods_requiring_spider:
+            kwargs["spider"] = self._spider
+
+        return method(**kwargs)
+
+    async def _process_request(
+        self,
+        request: Request,
+        download_func: Callable[[Request], Coroutine[Any, Any, Response]],
+    ) -> Response | Request:
         for method in self.methods["process_request"]:
             method = cast("Callable", method)
-                if method in self._mw_methods_requiring_spider:
             response = await ensure_awaitable(
-                        method(request=request, spider=self._spider),
+                self._handle_mw_method(method, request=request),
                 _warn=global_object_name(method),
             )
-                else:
+            if response is not None and not isinstance(response, (Response, Request)):
-                    response = await ensure_awaitable(
-                        method(request=request), _warn=global_object_name(method)
-                    )
-                if response is not None and not isinstance(
-                    response, (Response, Request)
-                ):
                 raise _InvalidOutput(
                     f"Middleware {method.__qualname__} must return None, Response or "
                     f"Request, got {response.__class__.__name__}"
@@ -98,7 +113,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 return response
         return await download_func(request)
 
-        async def process_response(response: Response | Request) -> Response | Request:
+    async def _process_response(
+        self, response: Response | Request, request: Request
+    ) -> Response | Request:
         if response is None:
             raise TypeError("Received None in process_response")
         if isinstance(response, Request):
@@ -106,16 +123,11 @@ class DownloaderMiddlewareManager(MiddlewareManager):
 
         for method in self.methods["process_response"]:
             method = cast("Callable", method)
-                if method in self._mw_methods_requiring_spider:
             response = await ensure_awaitable(
-                        method(request=request, response=response, spider=self._spider),
+                self._handle_mw_method(method, request=request, response=response),
-                        _warn=global_object_name(method),
-                    )
-                else:
-                    response = await ensure_awaitable(
-                        method(request=request, response=response),
                 _warn=global_object_name(method),
             )
+
             if not isinstance(response, (Response, Request)):
                 raise _InvalidOutput(
                     f"Middleware {method.__qualname__} must return Response or Request, "
@@ -125,24 +137,16 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                 return response
         return response
 
-        async def process_exception(exception: Exception) -> Response | Request:
+    async def _process_exception(
+        self, exception: Exception, request: Request | Response
+    ) -> Response | Request:
         for method in self.methods["process_exception"]:
             method = cast("Callable", method)
-                if method in self._mw_methods_requiring_spider:
             response = await ensure_awaitable(
-                        method(
+                self._handle_mw_method(method, request=request, exception=exception),
-                            request=request, exception=exception, spider=self._spider
-                        ),
                 _warn=global_object_name(method),
             )
-                else:
+            if response is not None and not isinstance(response, (Response, Request)):
-                    response = await ensure_awaitable(
-                        method(request=request, exception=exception),
-                        _warn=global_object_name(method),
-                    )
-                if response is not None and not isinstance(
-                    response, (Response, Request)
-                ):
                 raise _InvalidOutput(
                     f"Middleware {method.__qualname__} must return None, Response or "
                     f"Request, got {type(response)}"
@@ -150,12 +154,3 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             if response:
                 return response
         raise exception
-
-        try:
-            result: Response | Request = await process_request(request)
-        except Exception as ex:
-            await _defer_sleep_async()
-            # either returns a request or response (which we pass to process_response())
-            # or reraises the exception
-            result = await process_exception(ex)
-        return await process_response(result)

scrapy/core/engine.py

@@ -11,6 +11,7 @@ import asyncio
 import contextlib
 import logging
 import warnings
+from functools import partial
 from time import time
 from traceback import format_exc
 from typing import TYPE_CHECKING, Any
@@ -273,7 +274,7 @@ class ExecutionEngine:
         """
         assert self._start is not None
         try:
-            item_or_request = await self._start.__anext__()
+            item_or_request = await anext(self._start)
         except StopAsyncIteration:
             self._start = None
         except Exception as exception:
@@ -352,6 +353,10 @@ class ExecutionEngine:
             or self.scraper.slot.needs_backout()
         )
 
+    def _remove_request(self, _: Any, request: Request) -> None:
+        assert self._slot
+        self._slot.remove_request(request)
+
     def _start_scheduled_request(self) -> bool:
         assert self._slot is not None  # typing
         assert self.spider is not None  # typing
@@ -371,11 +376,7 @@ class ExecutionEngine:
             )
         )
 
-        def _remove_request(_: Any) -> None:
+        d2: Deferred[None] = d.addBoth(partial(self._remove_request, request=request))
-            assert self._slot
-            self._slot.remove_request(request)
-
-        d2: Deferred[None] = d.addBoth(_remove_request)
         d2.addErrback(
             lambda f: logger.info(
                 "Error while removing request from slot",
@@ -612,30 +613,33 @@ class ExecutionEngine:
             "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider}
         )
 
-        def log_failure(msg: str) -> None:
-            logger.error(msg, exc_info=True, extra={"spider": spider})  # noqa: LOG014
-
         try:
             await self._slot.close()
         except Exception:
-            log_failure("Slot close failure")
+            logger.error("Slot close failure", exc_info=True, extra={"spider": spider})
 
         try:
             self.downloader.close()
         except Exception:
-            log_failure("Downloader close failure")
+            logger.error(
+                "Downloader close failure", exc_info=True, extra={"spider": spider}
+            )
 
         try:
             await self.scraper.close_spider_async()
         except Exception:
-            log_failure("Scraper close failure")
+            logger.error(
+                "Scraper close failure", exc_info=True, extra={"spider": spider}
+            )
 
         if hasattr(self._slot.scheduler, "close"):
             try:
                 if (d := self._slot.scheduler.close(reason)) is not None:
                     await maybe_deferred_to_future(d)
             except Exception:
-                log_failure("Scheduler close failure")
+                logger.error(
+                    "Scheduler close failure", exc_info=True, extra={"spider": spider}
+                )
 
         try:
             await self.signals.send_catch_log_async(
@@ -644,7 +648,11 @@ class ExecutionEngine:
                 reason=reason,
             )
         except Exception:
-            log_failure("Error while sending spider_close signal")
+            logger.error(
+                "Error while sending spider_close signal",
+                exc_info=True,
+                extra={"spider": spider},
+            )
 
         assert self.crawler.stats
         try:
@@ -661,7 +669,7 @@ class ExecutionEngine:
             else:
                 self.crawler.stats.close_spider(reason=reason)
         except Exception:
-            log_failure("Stats close failure")
+            logger.error("Stats close failure")
 
         logger.info(
             "Spider closed (%(reason)s)",
@@ -675,4 +683,4 @@ class ExecutionEngine:
         try:
             await ensure_awaitable(self._spider_closed_callback(spider))
         except Exception:
-            log_failure("Error running spider_closed_callback")
+            logger.error("Error running spider_closed_callback")

scrapy/core/http2/stream.py

@@ -155,7 +155,9 @@ class Stream:
             "status": None,
         }
 
-        def _cancel(_: Any) -> None:
+        self._deferred_response: Deferred[Response] = Deferred(self._cancel)
+
+    def _cancel(self, _: Any) -> None:
         # Close this stream as gracefully as possible
         # If the associated request is initiated we reset this stream
         # else we directly call close() method
@@ -164,8 +166,6 @@ class Stream:
         else:
             self.close(StreamCloseReason.CANCELLED)
 
-        self._deferred_response: Deferred[Response] = Deferred(_cancel)
-
     def __repr__(self) -> str:
         return f"Stream(id={self.stream_id!r})"
 

scrapy/core/spidermw.py

@@ -112,41 +112,60 @@ class SpiderMiddlewareManager(MiddlewareManager):
         exception_processor_index: int,
         recover_to: MutableChain[_T] | MutableAsyncChain[_T],
     ) -> Iterable[_T] | AsyncIterator[_T]:
-        def process_sync(iterable: Iterable[_T]) -> Iterable[_T]:
+
+        if isinstance(iterable, AsyncIterator):
+            return self._process_async(
+                response,
+                iterable,
+                exception_processor_index,
+                cast("MutableAsyncChain[_T]", recover_to),
+            )
+        return self._process_sync(
+            response,
+            iterable,
+            exception_processor_index,
+            cast("MutableChain[_T]", recover_to),
+        )
+
+    def _process_sync(
+        self,
+        response: Response,
+        iterable: Iterable[_T],
+        exception_processor_index: int,
+        recover_to: MutableChain[_T],
+    ) -> Iterable[_T]:
         try:
             yield from iterable
         except Exception as ex:
             exception_result = cast(
                 "Failure | MutableChain[_T]",
-                    self._process_spider_exception(
+                self._process_spider_exception(response, ex, exception_processor_index),
-                        response, ex, exception_processor_index
-                    ),
             )
             if isinstance(exception_result, Failure):
                 raise
             assert isinstance(recover_to, MutableChain)
             recover_to.extend(exception_result)
 
-        async def process_async(iterable: AsyncIterator[_T]) -> AsyncIterator[_T]:
+    async def _process_async(
+        self,
+        response: Response,
+        iterable: AsyncIterator[_T],
+        exception_processor_index: int,
+        recover_to: MutableAsyncChain[_T],
+    ) -> AsyncIterator[_T]:
         try:
             async for r in iterable:
                 yield r
         except Exception as ex:
             exception_result = cast(
                 "Failure | MutableAsyncChain[_T]",
-                    self._process_spider_exception(
+                self._process_spider_exception(response, ex, exception_processor_index),
-                        response, ex, exception_processor_index
-                    ),
             )
             if isinstance(exception_result, Failure):
                 raise
             assert isinstance(recover_to, MutableAsyncChain)
             recover_to.extend(exception_result)
 
-        if isinstance(iterable, AsyncIterator):
-            return process_async(iterable)
-        return process_sync(iterable)
-
     def _process_spider_exception(
         self,
         response: Response,
@@ -350,25 +369,14 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 "scrape_response_async() called on a SpiderMiddlewareManager"
                 " instance created without a crawler."
             )
-
-        async def process_callback_output(
-            result: Iterable[_T] | AsyncIterator[_T],
-        ) -> MutableChain[_T] | MutableAsyncChain[_T]:
-            return await self._process_callback_output(response, result)
-
-        def process_spider_exception(
-            exception: Exception,
-        ) -> MutableChain[_T] | MutableAsyncChain[_T]:
-            return self._process_spider_exception(response, exception)
-
         try:
             it: Iterable[_T] | AsyncIterator[_T] = await self._process_spider_input(
                 scrape_func, response, request
             )
-            return await process_callback_output(it)
+            return await self._process_callback_output(response, it)
         except Exception as ex:
             await _defer_sleep_async()
-            return process_spider_exception(ex)
+            return self._process_spider_exception(response, ex)
 
     async def process_start(
         self, spider: Spider | None = None

scrapy/crawler.py

@@ -7,6 +7,7 @@ import pprint
 import signal
 import warnings
 from abc import ABC, abstractmethod
+from functools import partial
 from typing import TYPE_CHECKING, Any, TypeVar
 
 from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
@@ -568,28 +569,30 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
         crawler = self.create_crawler(crawler_or_spidercls)
         return self._crawl(crawler, *args, **kwargs)
 
-    def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]:
+    async def _crawl_and_track(
-        # At this point the asyncio loop has been installed either by the user
+        self, crawler: Crawler, *args: Any, **kwargs: Any
-        # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()).
+    ) -> None:
-        loop = asyncio.get_event_loop()
-        self.crawlers.add(crawler)
-
-        async def _crawl_and_track() -> None:
         try:
             await crawler.crawl_async(*args, **kwargs)
         except Exception:
             self.bootstrap_failed = True
             raise  # re-raise so asyncio still logs it to stderr naturally
 
-        task = loop.create_task(_crawl_and_track())
+    def _done(self, task: asyncio.Task[None], crawler: Crawler) -> None:
-        self._active.add(task)
-
-        def _done(_: asyncio.Task[None]) -> None:
-            self.crawlers.discard(crawler)
         self._active.discard(task)
+        self.crawlers.discard(crawler)
         self.bootstrap_failed |= not getattr(crawler, "spider", None)
 
-        task.add_done_callback(_done)
+    def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]:
+        # At this point the asyncio loop has been installed either by the user
+        # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()).
+        loop = asyncio.get_event_loop()
+        self.crawlers.add(crawler)
+
+        task = loop.create_task(self._crawl_and_track(crawler, *args, **kwargs))
+        self._active.add(task)
+        task.add_done_callback(partial(self._done, crawler=crawler))
+
         return task
 
     async def stop(self) -> None:
@@ -1007,15 +1010,16 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
         if (loop := self._reactorless_loop) is None:
             return
 
-        def _create_shutdown_task() -> None:
+        loop.call_soon_threadsafe(self._create_shutdown_task)
+
+    def _create_shutdown_task(self) -> None:
+        assert self._reactorless_loop
         coro = self._shutdown_graceful_reactorless()
         try:
-                loop.create_task(coro)
+            self._reactorless_loop.create_task(coro)
         except RuntimeError:
             coro.close()
 
-        loop.call_soon_threadsafe(_create_shutdown_task)
-
     async def _shutdown_graceful_reactorless(self) -> None:
         await self.stop()
         if not self._stop_after_crawl:

scrapy/extensions/feedexport.py

@@ -534,14 +534,16 @@ class FeedExporter:
         # Send FEED_EXPORTER_CLOSED signal
         await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed)
 
-    async def _close_slot(self, slot: FeedSlot, spider: Spider) -> None:
+    @staticmethod
-        def get_file(slot_: FeedSlot) -> IO[bytes]:
+    def _get_file(slot_: FeedSlot) -> IO[bytes]:
         assert slot_.file
         if isinstance(slot_.file, PostProcessingManager):
             slot_.file.close()
             return slot_.file.file
         return slot_.file
 
+    async def _close_slot(self, slot: FeedSlot, spider: Spider) -> None:
+
         if slot.itemcount:
             # Normal case
             slot.finish_exporting()
@@ -557,7 +559,7 @@ class FeedExporter:
         slot_type = type(slot.storage).__name__
         assert self.crawler.stats
         try:
-            await ensure_awaitable(slot.storage.store(get_file(slot)))
+            await ensure_awaitable(slot.storage.store(self._get_file(slot)))
         except Exception:
             logger.error(
                 "Error storing %s",

scrapy/pipelines/__init__.py

@@ -37,16 +37,16 @@ class ItemPipelineManager(MiddlewareManager):
             settings.get_component_priority_dict_with_base("ITEM_PIPELINES")
         )
 
-    def _add_middleware(self, pipe: Any) -> None:
+    def _add_middleware(self, mw: Any) -> None:
-        if hasattr(pipe, "open_spider"):
+        if hasattr(mw, "open_spider"):
-            self.methods["open_spider"].append(pipe.open_spider)
+            self.methods["open_spider"].append(mw.open_spider)
-            self._check_mw_method_spider_arg(pipe.open_spider)
+            self._check_mw_method_spider_arg(mw.open_spider)
-        if hasattr(pipe, "close_spider"):
+        if hasattr(mw, "close_spider"):
-            self.methods["close_spider"].appendleft(pipe.close_spider)
+            self.methods["close_spider"].appendleft(mw.close_spider)
-            self._check_mw_method_spider_arg(pipe.close_spider)
+            self._check_mw_method_spider_arg(mw.close_spider)
-        if hasattr(pipe, "process_item"):
+        if hasattr(mw, "process_item"):
-            self.methods["process_item"].append(pipe.process_item)
+            self.methods["process_item"].append(mw.process_item)
-            self._check_mw_method_spider_arg(pipe.process_item)
+            self._check_mw_method_spider_arg(mw.process_item)
 
     def process_item(self, item: Any, spider: Spider) -> Deferred[Any]:
         warnings.warn(
@@ -62,32 +62,44 @@ class ItemPipelineManager(MiddlewareManager):
             "process_item", item, add_spider=True, warn_deferred=True
         )
 
-    def _process_parallel_dfd(self, methodname: str) -> Deferred[list[None]]:
+    def _get_dfd(
-        methods = cast(
+        self,
-            "Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]",
-            self.methods[methodname],
-        )
-
-        def get_dfd(
         method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
     ) -> Deferred[None]:
         if method in self._mw_methods_requiring_spider:
             return _maybeDeferred_coro(method, True, self._spider)
         return _maybeDeferred_coro(method, True)
 
-        dfds = [get_dfd(m) for m in methods]
+    @staticmethod
+    def _eb(failure: Failure) -> Failure:
+        assert isinstance(failure.value, FirstError)
+        return failure.value.subFailure
+
+    def _process_parallel_dfd(self, methodname: str) -> Deferred[list[None]]:
+        methods = cast(
+            "Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]",
+            self.methods[methodname],
+        )
+
+        dfds = [self._get_dfd(m) for m in methods]
         d: Deferred[list[tuple[bool, None]]] = DeferredList(
             dfds, fireOnOneErrback=True, consumeErrors=True
         )
         d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r])
 
-        def eb(failure: Failure) -> Failure:
+        d2.addErrback(self._eb)
-            assert isinstance(failure.value, FirstError)
-            return failure.value.subFailure
-
-        d2.addErrback(eb)
         return d2
 
+    def get_awaitable(
+        self,
+        method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
+    ) -> Awaitable[None]:
+        if method in self._mw_methods_requiring_spider:
+            result = method(self._spider)
+        else:
+            result = method()
+        return ensure_awaitable(result, _warn=global_object_name(method))
+
     async def _process_parallel_asyncio(self, methodname: str) -> list[None]:
         methods = cast(
             "Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]",
@@ -96,16 +108,7 @@ class ItemPipelineManager(MiddlewareManager):
         if not methods:
             return []
 
-        def get_awaitable(
+        awaitables = [self.get_awaitable(m) for m in methods]
-            method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
-        ) -> Awaitable[None]:
-            if method in self._mw_methods_requiring_spider:
-                result = method(self._spider)
-            else:
-                result = method()
-            return ensure_awaitable(result, _warn=global_object_name(method))
-
-        awaitables = [get_awaitable(m) for m in methods]
         await asyncio.gather(*awaitables)
         return [None for _ in methods]
 

scrapy/pipelines/files.py

@@ -186,16 +186,18 @@ class S3FilesStore:
             raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
         self.bucket, self.prefix = uri[5:].split("/", 1)
 
-    def stat_file(
+    @staticmethod
-        self, path: str, info: MediaPipeline.SpiderInfo
-    ) -> Deferred[StatInfo]:
     def _onsuccess(boto_key: dict[str, Any]) -> StatInfo:
         checksum = boto_key["ETag"].strip('"')
         last_modified = boto_key["LastModified"]
         modified_stamp = time.mktime(last_modified.timetuple())
         return {"checksum": checksum, "last_modified": modified_stamp}
 
-        return self._get_boto_key(path).addCallback(_onsuccess)
+    def stat_file(
+        self, path: str, info: MediaPipeline.SpiderInfo
+    ) -> Deferred[StatInfo]:
+
+        return self._get_boto_key(path).addCallback(self._onsuccess)
 
     def _get_boto_key(self, path: str) -> Deferred[dict[str, Any]]:
         key_name = f"{self.prefix}{path}"
@@ -308,9 +310,7 @@ class GCSFilesStore:
                 {"bucket": bucket},
             )
 
-    def stat_file(
+    @staticmethod
-        self, path: str, info: MediaPipeline.SpiderInfo
-    ) -> Deferred[StatInfo]:
     def _onsuccess(blob: Any) -> StatInfo:
         if blob:
             checksum = base64.b64decode(blob.md5_hash).hex()
@@ -318,10 +318,14 @@ class GCSFilesStore:
             return {"checksum": checksum, "last_modified": last_modified}
         return {}
 
+    def stat_file(
+        self, path: str, info: MediaPipeline.SpiderInfo
+    ) -> Deferred[StatInfo]:
+
         blob_path = self._get_blob_path(path)
         return deferred_from_coro(
             run_in_thread(self.bucket.get_blob, blob_path)
-        ).addCallback(_onsuccess)
+        ).addCallback(self._onsuccess)
 
     def _get_content_type(self, headers: dict[str, str] | None) -> str:
         if headers and "Content-Type" in headers:
@@ -395,10 +399,7 @@ class FTPFilesStore:
             )
         )
 
-    def stat_file(
+    def _stat_file(self, path: str) -> StatInfo:
-        self, path: str, info: MediaPipeline.SpiderInfo
-    ) -> Deferred[StatInfo]:
-        def _stat_file(path: str) -> StatInfo:
         try:
             with FTP() as ftp:
                 ftp.connect(self.host, self.port)
@@ -414,7 +415,10 @@ class FTPFilesStore:
         except Exception:
             return {}
 
-        return deferred_from_coro(run_in_thread(_stat_file, path))
+    def stat_file(
+        self, path: str, info: MediaPipeline.SpiderInfo
+    ) -> Deferred[StatInfo]:
+        return deferred_from_coro(run_in_thread(self._stat_file, path))
 
 
 class FilesPipeline(MediaPipeline):
@@ -534,10 +538,13 @@ class FilesPipeline(MediaPipeline):
         store_cls = self.STORE_SCHEMES[scheme]
         return store_cls(uri)
 
-    def media_to_download(
+    def _onsuccess(
-        self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None
+        self,
-    ) -> Deferred[FileInfo | None] | None:
+        result: StatInfo,
-        def _onsuccess(result: StatInfo) -> FileInfo | None:
+        request: Request,
+        info: MediaPipeline.SpiderInfo,
+        path: str,
+    ) -> FileInfo | None:
         if not result:
             return None  # returning None force download
 
@@ -567,10 +574,15 @@ class FilesPipeline(MediaPipeline):
             "status": "uptodate",
         }
 
+    def media_to_download(
+        self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None
+    ) -> Deferred[FileInfo | None] | None:
         path = self.file_path(request, info=info, item=item)
         # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
         dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info)  # type: ignore[call-overload]
-        dfd2: Deferred[FileInfo | None] = dfd.addCallback(_onsuccess)
+        dfd2: Deferred[FileInfo | None] = dfd.addCallback(
+            functools.partial(self._onsuccess, request=request, info=info, path=path)
+        )
         dfd2.addErrback(lambda _: None)
         dfd2.addErrback(
             lambda f: logger.error(

scrapy/settings/__init__.py

@@ -143,7 +143,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
             raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).")
         self.set(name, [v for v in value if v != item], self.getpriority(name) or 0)
 
-    def get(self, name: _SettingsKey, default: Any = None) -> Any:
+    def get(self, name: _SettingsKey, default: Any = None) -> Any:  # pylint: disable=arguments-renamed
         """
         Get a setting value without affecting its original type.
 
@@ -510,7 +510,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
         component_priority_dict[cls] = priority
         self.set(name, component_priority_dict, self.getpriority(name) or 0)
 
-    def setdefault(
+    def setdefault(  # pylint: disable=arguments-renamed
         self,
         name: _SettingsKey,
         default: Any = None,
@@ -691,14 +691,14 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
         else:
             p.text(pformat(self.copy_to_dict()))
 
-    def pop(self, name: _SettingsKey, default: Any = __default) -> Any:
+    def pop(self, name: _SettingsKey, default: Any = __default) -> Any:  # pylint: disable=arguments-renamed
         try:
             value = self.attributes[name].value
         except KeyError:
             if default is self.__default:
                 raise
             return default
-        self.__delitem__(name)
+        del self[name]
         return value
 
 

scrapy/utils/benchserver.py

@@ -9,7 +9,7 @@ from twisted.web.server import Request, Site
 class Root(Resource):
     isLeaf = True
 
-    def getChild(self, name: str, request: Request) -> Resource:
+    def getChild(self, path: str, request: Request) -> Resource:
         return self
 
     def render(self, request: Request) -> bytes:

scrapy/utils/defer.py

@@ -262,7 +262,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
     def _call_anext(self) -> None:
         # This starts waiting for the next result from aiterator.
         # If aiterator is exhausted, _errback will be called.
-        self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
+        self.anext_deferred = deferred_from_coro(anext(self.aiterator))
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
     def __next__(self) -> Deferred[Any]:
@@ -370,10 +370,10 @@ async def aiter_errback(
     """Wrap an async iterable calling an errback if an error is caught while
     iterating it. Similar to :func:`scrapy.utils.defer.iter_errback`.
     """
-    it = aiterable.__aiter__()
+    it = aiter(aiterable)
     while True:
         try:
-            yield await it.__anext__()
+            yield await anext(it)
         except StopAsyncIteration:
             break
         except Exception:

scrapy/utils/misc.py

@@ -255,6 +255,11 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
 _generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
 
 
+def _returns_none(return_node: ast.Return) -> bool:
+    value = return_node.value
+    return value is None or (isinstance(value, ast.Constant) and value.value is None)
+
+
 def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:  # noqa: A002
     """
     Returns True if a callable is a generator function which includes a
@@ -263,12 +268,6 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:  # noq
     if callable in _generator_callbacks_cache:
         return bool(_generator_callbacks_cache[callable])
 
-    def returns_none(return_node: ast.Return) -> bool:
-        value = return_node.value
-        return value is None or (
-            isinstance(value, ast.Constant) and value.value is None
-        )
-
     if inspect.isgeneratorfunction(callable):
         func = callable
         while isinstance(func, partial):
@@ -284,7 +283,7 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:  # noq
 
         tree = ast.parse(code)
         for node in walk_callable(tree):
-            if isinstance(node, ast.Return) and not returns_none(node):
+            if isinstance(node, ast.Return) and not _returns_none(node):
                 _generator_callbacks_cache[callable] = True
                 return bool(_generator_callbacks_cache[callable])
 

scrapy/utils/python.py

@@ -99,6 +99,16 @@ def to_bytes(
     return text.encode(encoding, errors)
 
 
+def _chunk_iter(text: str, chunk_size: int) -> Iterable[tuple[str, int]]:
+    offset = len(text)
+    while True:
+        offset -= chunk_size * 1024
+        if offset <= 0:
+            break
+        yield (text[offset:], offset)
+    yield (text, 0)
+
+
 def re_rsearch(
     pattern: str | Pattern[str], text: str, chunk_size: int = 1024
 ) -> tuple[int, int] | None:
@@ -115,19 +125,10 @@ def re_rsearch(
     the start position of the match, and the ending (regarding the entire text).
     """
 
-    def _chunk_iter() -> Iterable[tuple[str, int]]:
-        offset = len(text)
-        while True:
-            offset -= chunk_size * 1024
-            if offset <= 0:
-                break
-            yield (text[offset:], offset)
-        yield (text, 0)
-
     if isinstance(pattern, str):
         pattern = re.compile(pattern)
 
-    for chunk, offset in _chunk_iter():
+    for chunk, offset in _chunk_iter(text, chunk_size):
         matches = list(pattern.finditer(chunk))
         if matches:
             start, end = matches[-1].span()

tests/mockserver/http.py

@@ -85,7 +85,7 @@ class Root(resource.Resource):
         self.putChild(b"response-headers", ResponseHeadersResource())
         self.putChild(b"set-cookie", SetCookie())
 
-    def getChild(self, name, request):
+    def getChild(self, path, request):
         return self
 
     def render(self, request):

tests/mockserver/simple_https.py

@@ -13,7 +13,7 @@ class Root(resource.Resource):
         resource.Resource.__init__(self)
         self.putChild(b"file", Data(b"0123456789", "text/plain"))
 
-    def getChild(self, name, request):
+    def getChild(self, path, request):
         return self
 
 

tox.ini

@@ -89,8 +89,9 @@ basepython = python3
 deps =
     {[testenv:extra-deps]deps}
     pylint==4.0.2
+    pylint-per-file-ignores # https://github.com/pylint-dev/pylint/issues/3767#issuecomment-1319916278
 commands =
-    pylint conftest.py docs extras scrapy tests
+    pylint {posargs:conftest.py docs extras scrapy tests}
 
 [testenv:twinecheck]
 basepython = python3