diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py index 734614aa5..569438574 100755 --- a/extras/qps-bench-server.py +++ b/extras/qps-bench-server.py @@ -18,7 +18,7 @@ class Root(Resource): self.tail.clear() self.start = self.lastmark = self.lasttime = time() - def getChild(self, request, name): + def getChild(self, path, request): return self def render(self, request): diff --git a/pyproject.toml b/pyproject.toml index 2132c5c50..a68222b36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -185,6 +185,7 @@ jobs = 1 # >1 hides results extension-pkg-allow-list=[ "lxml", ] +load-plugins = ["pylint_per_file_ignores"] [tool.pylint."MESSAGES CONTROL"] enable = [ @@ -244,15 +245,13 @@ disable = [ "unused-import", # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix") - "abstract-method", "arguments-differ", - "arguments-renamed", - "dangerous-default-value", "keyword-arg-before-vararg", - "pointless-statement", - "raise-missing-from", - "unnecessary-dunder-call", - "used-before-assignment", +] +# requires `pylint_per_file_ignores` plugin +per-file-ignores = [ + # Extended list of ones that we may want to address, only for tests + "./tests/*:abstract-method,arguments-renamed,dangerous-default-value,pointless-statement,raise-missing-from,unnecessary-dunder-call,used-before-assignment", ] [tool.pytest.ini_options] diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index f84bbe0d7..24f56b7a6 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -125,7 +125,7 @@ class ContractsManager: def from_spider(self, spider: Spider, results: TestResult) -> list[Request | None]: requests: list[Request | None] = [] for method in self.tested_methods_from_spidercls(type(spider)): - bound_method = spider.__getattribute__(method) + bound_method = getattr(spider, method) try: requests.append(self.from_method(bound_method, results)) except Exception: diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 439638d33..975078803 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -6,6 +6,7 @@ import ipaddress import logging import re from contextlib import suppress +from functools import partial from io import BytesIO from time import monotonic from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast @@ -547,11 +548,7 @@ class ScrapyAgent: get_warnsize_msg(expected_size, warnsize, request, expected=True) ) - def _cancel(_: Any) -> None: - # Abort connection immediately. - txresponse._transport._producer.abortConnection() - - d: Deferred[_ResultT] = Deferred(_cancel) + d: Deferred[_ResultT] = Deferred(partial(self._cancel, txresponse=txresponse)) txresponse.deliverBody( _ResponseReader( finished=d, @@ -570,6 +567,11 @@ class ScrapyAgent: return d + @staticmethod + def _cancel(_: Any, txresponse: TxResponse) -> None: + # Abort connection immediately. + txresponse._transport._producer.abortConnection() + def _cb_bodydone(self, result: _ResultT, url: str) -> Response: headers = self._headers_from_twisted_response(result["txresponse"]) try: @@ -667,17 +669,17 @@ class _ResponseReader(Protocol): assert hostname is not None _log_ssl_conn_debug_info(hostname, connection) - def dataReceived(self, bodyBytes: bytes) -> None: + def dataReceived(self, data: bytes) -> None: # This maybe called several times after cancel was called with buffered data. if self._finished.called: return assert self.transport - self._bodybuf.write(bodyBytes) - self._bytes_received += len(bodyBytes) + self._bodybuf.write(data) + self._bytes_received += len(data) if stop_download := check_stop_download( - signals.bytes_received, self._crawler, self._request, data=bodyBytes + signals.bytes_received, self._crawler, self._request, data=data ): self.transport.stopProducing() self.transport.loseConnection() diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index d08c6f8e3..6685d90af 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -75,87 +75,82 @@ class DownloaderMiddlewareManager(MiddlewareManager): download_func: Callable[[Request], Coroutine[Any, Any, Response]], request: Request, ) -> Response | Request: - async def process_request(request: Request) -> Response | Request: - for method in self.methods["process_request"]: - method = cast("Callable", method) - if method in self._mw_methods_requiring_spider: - response = await ensure_awaitable( - method(request=request, spider=self._spider), - _warn=global_object_name(method), - ) - else: - response = await ensure_awaitable( - method(request=request), _warn=global_object_name(method) - ) - if response is not None and not isinstance( - response, (Response, Request) - ): - raise _InvalidOutput( - f"Middleware {method.__qualname__} must return None, Response or " - f"Request, got {response.__class__.__name__}" - ) - if response: - return response - return await download_func(request) - - async def process_response(response: Response | Request) -> Response | Request: - if response is None: - raise TypeError("Received None in process_response") - if isinstance(response, Request): - return response - - for method in self.methods["process_response"]: - method = cast("Callable", method) - if method in self._mw_methods_requiring_spider: - response = await ensure_awaitable( - method(request=request, response=response, spider=self._spider), - _warn=global_object_name(method), - ) - else: - response = await ensure_awaitable( - method(request=request, response=response), - _warn=global_object_name(method), - ) - if not isinstance(response, (Response, Request)): - raise _InvalidOutput( - f"Middleware {method.__qualname__} must return Response or Request, " - f"got {type(response)}" - ) - if isinstance(response, Request): - return response - return response - - async def process_exception(exception: Exception) -> Response | Request: - for method in self.methods["process_exception"]: - method = cast("Callable", method) - if method in self._mw_methods_requiring_spider: - response = await ensure_awaitable( - method( - request=request, exception=exception, spider=self._spider - ), - _warn=global_object_name(method), - ) - else: - response = await ensure_awaitable( - method(request=request, exception=exception), - _warn=global_object_name(method), - ) - if response is not None and not isinstance( - response, (Response, Request) - ): - raise _InvalidOutput( - f"Middleware {method.__qualname__} must return None, Response or " - f"Request, got {type(response)}" - ) - if response: - return response - raise exception try: - result: Response | Request = await process_request(request) + result: Response | Request = await self._process_request( + request, download_func + ) except Exception as ex: await _defer_sleep_async() # either returns a request or response (which we pass to process_response()) # or reraises the exception - result = await process_exception(ex) - return await process_response(result) + result = await self._process_exception(ex, request) + return await self._process_response(result, request) + + def _handle_mw_method(self, method: Callable, **kwargs: Any) -> Any: + if method in self._mw_methods_requiring_spider: + kwargs["spider"] = self._spider + + return method(**kwargs) + + async def _process_request( + self, + request: Request, + download_func: Callable[[Request], Coroutine[Any, Any, Response]], + ) -> Response | Request: + for method in self.methods["process_request"]: + method = cast("Callable", method) + response = await ensure_awaitable( + self._handle_mw_method(method, request=request), + _warn=global_object_name(method), + ) + if response is not None and not isinstance(response, (Response, Request)): + raise _InvalidOutput( + f"Middleware {method.__qualname__} must return None, Response or " + f"Request, got {response.__class__.__name__}" + ) + if response: + return response + return await download_func(request) + + async def _process_response( + self, response: Response | Request, request: Request + ) -> Response | Request: + if response is None: + raise TypeError("Received None in process_response") + if isinstance(response, Request): + return response + + for method in self.methods["process_response"]: + method = cast("Callable", method) + response = await ensure_awaitable( + self._handle_mw_method(method, request=request, response=response), + _warn=global_object_name(method), + ) + + if not isinstance(response, (Response, Request)): + raise _InvalidOutput( + f"Middleware {method.__qualname__} must return Response or Request, " + f"got {type(response)}" + ) + if isinstance(response, Request): + return response + return response + + async def _process_exception( + self, exception: Exception, request: Request | Response + ) -> Response | Request: + for method in self.methods["process_exception"]: + method = cast("Callable", method) + response = await ensure_awaitable( + self._handle_mw_method(method, request=request, exception=exception), + _warn=global_object_name(method), + ) + if response is not None and not isinstance(response, (Response, Request)): + raise _InvalidOutput( + f"Middleware {method.__qualname__} must return None, Response or " + f"Request, got {type(response)}" + ) + if response: + return response + raise exception diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 94920e840..1033e874f 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -11,6 +11,7 @@ import asyncio import contextlib import logging import warnings +from functools import partial from time import time from traceback import format_exc from typing import TYPE_CHECKING, Any @@ -273,7 +274,7 @@ class ExecutionEngine: """ assert self._start is not None try: - item_or_request = await self._start.__anext__() + item_or_request = await anext(self._start) except StopAsyncIteration: self._start = None except Exception as exception: @@ -352,6 +353,10 @@ class ExecutionEngine: or self.scraper.slot.needs_backout() ) + def _remove_request(self, _: Any, request: Request) -> None: + assert self._slot + self._slot.remove_request(request) + def _start_scheduled_request(self) -> bool: assert self._slot is not None # typing assert self.spider is not None # typing @@ -371,11 +376,7 @@ class ExecutionEngine: ) ) - def _remove_request(_: Any) -> None: - assert self._slot - self._slot.remove_request(request) - - d2: Deferred[None] = d.addBoth(_remove_request) + d2: Deferred[None] = d.addBoth(partial(self._remove_request, request=request)) d2.addErrback( lambda f: logger.info( "Error while removing request from slot", @@ -612,30 +613,33 @@ class ExecutionEngine: "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider} ) - def log_failure(msg: str) -> None: - logger.error(msg, exc_info=True, extra={"spider": spider}) # noqa: LOG014 - try: await self._slot.close() except Exception: - log_failure("Slot close failure") + logger.error("Slot close failure", exc_info=True, extra={"spider": spider}) try: self.downloader.close() except Exception: - log_failure("Downloader close failure") + logger.error( + "Downloader close failure", exc_info=True, extra={"spider": spider} + ) try: await self.scraper.close_spider_async() except Exception: - log_failure("Scraper close failure") + logger.error( + "Scraper close failure", exc_info=True, extra={"spider": spider} + ) if hasattr(self._slot.scheduler, "close"): try: if (d := self._slot.scheduler.close(reason)) is not None: await maybe_deferred_to_future(d) except Exception: - log_failure("Scheduler close failure") + logger.error( + "Scheduler close failure", exc_info=True, extra={"spider": spider} + ) try: await self.signals.send_catch_log_async( @@ -644,7 +648,11 @@ class ExecutionEngine: reason=reason, ) except Exception: - log_failure("Error while sending spider_close signal") + logger.error( + "Error while sending spider_close signal", + exc_info=True, + extra={"spider": spider}, + ) assert self.crawler.stats try: @@ -661,7 +669,7 @@ class ExecutionEngine: else: self.crawler.stats.close_spider(reason=reason) except Exception: - log_failure("Stats close failure") + logger.error("Stats close failure") logger.info( "Spider closed (%(reason)s)", @@ -675,4 +683,4 @@ class ExecutionEngine: try: await ensure_awaitable(self._spider_closed_callback(spider)) except Exception: - log_failure("Error running spider_closed_callback") + logger.error("Error running spider_closed_callback") diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index 71569b217..4d072c555 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -155,16 +155,16 @@ class Stream: "status": None, } - def _cancel(_: Any) -> None: - # Close this stream as gracefully as possible - # If the associated request is initiated we reset this stream - # else we directly call close() method - if self.metadata["request_sent"]: - self.reset_stream(StreamCloseReason.CANCELLED) - else: - self.close(StreamCloseReason.CANCELLED) + self._deferred_response: Deferred[Response] = Deferred(self._cancel) - self._deferred_response: Deferred[Response] = Deferred(_cancel) + def _cancel(self, _: Any) -> None: + # Close this stream as gracefully as possible + # If the associated request is initiated we reset this stream + # else we directly call close() method + if self.metadata["request_sent"]: + self.reset_stream(StreamCloseReason.CANCELLED) + else: + self.close(StreamCloseReason.CANCELLED) def __repr__(self) -> str: return f"Stream(id={self.stream_id!r})" diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 790317357..d48a65967 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -112,40 +112,59 @@ class SpiderMiddlewareManager(MiddlewareManager): exception_processor_index: int, recover_to: MutableChain[_T] | MutableAsyncChain[_T], ) -> Iterable[_T] | AsyncIterator[_T]: - def process_sync(iterable: Iterable[_T]) -> Iterable[_T]: - try: - yield from iterable - except Exception as ex: - exception_result = cast( - "Failure | MutableChain[_T]", - self._process_spider_exception( - response, ex, exception_processor_index - ), - ) - if isinstance(exception_result, Failure): - raise - assert isinstance(recover_to, MutableChain) - recover_to.extend(exception_result) - - async def process_async(iterable: AsyncIterator[_T]) -> AsyncIterator[_T]: - try: - async for r in iterable: - yield r - except Exception as ex: - exception_result = cast( - "Failure | MutableAsyncChain[_T]", - self._process_spider_exception( - response, ex, exception_processor_index - ), - ) - if isinstance(exception_result, Failure): - raise - assert isinstance(recover_to, MutableAsyncChain) - recover_to.extend(exception_result) if isinstance(iterable, AsyncIterator): - return process_async(iterable) - return process_sync(iterable) + return self._process_async( + response, + iterable, + exception_processor_index, + cast("MutableAsyncChain[_T]", recover_to), + ) + return self._process_sync( + response, + iterable, + exception_processor_index, + cast("MutableChain[_T]", recover_to), + ) + + def _process_sync( + self, + response: Response, + iterable: Iterable[_T], + exception_processor_index: int, + recover_to: MutableChain[_T], + ) -> Iterable[_T]: + try: + yield from iterable + except Exception as ex: + exception_result = cast( + "Failure | MutableChain[_T]", + self._process_spider_exception(response, ex, exception_processor_index), + ) + if isinstance(exception_result, Failure): + raise + assert isinstance(recover_to, MutableChain) + recover_to.extend(exception_result) + + async def _process_async( + self, + response: Response, + iterable: AsyncIterator[_T], + exception_processor_index: int, + recover_to: MutableAsyncChain[_T], + ) -> AsyncIterator[_T]: + try: + async for r in iterable: + yield r + except Exception as ex: + exception_result = cast( + "Failure | MutableAsyncChain[_T]", + self._process_spider_exception(response, ex, exception_processor_index), + ) + if isinstance(exception_result, Failure): + raise + assert isinstance(recover_to, MutableAsyncChain) + recover_to.extend(exception_result) def _process_spider_exception( self, @@ -350,25 +369,14 @@ class SpiderMiddlewareManager(MiddlewareManager): "scrape_response_async() called on a SpiderMiddlewareManager" " instance created without a crawler." ) - - async def process_callback_output( - result: Iterable[_T] | AsyncIterator[_T], - ) -> MutableChain[_T] | MutableAsyncChain[_T]: - return await self._process_callback_output(response, result) - - def process_spider_exception( - exception: Exception, - ) -> MutableChain[_T] | MutableAsyncChain[_T]: - return self._process_spider_exception(response, exception) - try: it: Iterable[_T] | AsyncIterator[_T] = await self._process_spider_input( scrape_func, response, request ) - return await process_callback_output(it) + return await self._process_callback_output(response, it) except Exception as ex: await _defer_sleep_async() - return process_spider_exception(ex) + return self._process_spider_exception(response, ex) async def process_start( self, spider: Spider | None = None diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 0a19e9985..8e3ae7879 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -7,6 +7,7 @@ import pprint import signal import warnings from abc import ABC, abstractmethod +from functools import partial from typing import TYPE_CHECKING, Any, TypeVar from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks @@ -568,28 +569,30 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) + async def _crawl_and_track( + self, crawler: Crawler, *args: Any, **kwargs: Any + ) -> None: + try: + await crawler.crawl_async(*args, **kwargs) + except Exception: + self.bootstrap_failed = True + raise # re-raise so asyncio still logs it to stderr naturally + + def _done(self, task: asyncio.Task[None], crawler: Crawler) -> None: + self._active.discard(task) + self.crawlers.discard(crawler) + self.bootstrap_failed |= not getattr(crawler, "spider", None) + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]: # At this point the asyncio loop has been installed either by the user # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()). loop = asyncio.get_event_loop() self.crawlers.add(crawler) - async def _crawl_and_track() -> None: - try: - await crawler.crawl_async(*args, **kwargs) - except Exception: - self.bootstrap_failed = True - raise # re-raise so asyncio still logs it to stderr naturally - - task = loop.create_task(_crawl_and_track()) + task = loop.create_task(self._crawl_and_track(crawler, *args, **kwargs)) self._active.add(task) + task.add_done_callback(partial(self._done, crawler=crawler)) - def _done(_: asyncio.Task[None]) -> None: - self.crawlers.discard(crawler) - self._active.discard(task) - self.bootstrap_failed |= not getattr(crawler, "spider", None) - - task.add_done_callback(_done) return task async def stop(self) -> None: @@ -1007,14 +1010,15 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): if (loop := self._reactorless_loop) is None: return - def _create_shutdown_task() -> None: - coro = self._shutdown_graceful_reactorless() - try: - loop.create_task(coro) - except RuntimeError: - coro.close() + loop.call_soon_threadsafe(self._create_shutdown_task) - loop.call_soon_threadsafe(_create_shutdown_task) + def _create_shutdown_task(self) -> None: + assert self._reactorless_loop + coro = self._shutdown_graceful_reactorless() + try: + self._reactorless_loop.create_task(coro) + except RuntimeError: + coro.close() async def _shutdown_graceful_reactorless(self) -> None: await self.stop() diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 8f70dbc2e..d0430d0db 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -534,13 +534,15 @@ class FeedExporter: # Send FEED_EXPORTER_CLOSED signal await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed) + @staticmethod + def _get_file(slot_: FeedSlot) -> IO[bytes]: + assert slot_.file + if isinstance(slot_.file, PostProcessingManager): + slot_.file.close() + return slot_.file.file + return slot_.file + async def _close_slot(self, slot: FeedSlot, spider: Spider) -> None: - def get_file(slot_: FeedSlot) -> IO[bytes]: - assert slot_.file - if isinstance(slot_.file, PostProcessingManager): - slot_.file.close() - return slot_.file.file - return slot_.file if slot.itemcount: # Normal case @@ -557,7 +559,7 @@ class FeedExporter: slot_type = type(slot.storage).__name__ assert self.crawler.stats try: - await ensure_awaitable(slot.storage.store(get_file(slot))) + await ensure_awaitable(slot.storage.store(self._get_file(slot))) except Exception: logger.error( "Error storing %s", diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 18473c534..84fb5f85e 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -37,16 +37,16 @@ class ItemPipelineManager(MiddlewareManager): settings.get_component_priority_dict_with_base("ITEM_PIPELINES") ) - def _add_middleware(self, pipe: Any) -> None: - if hasattr(pipe, "open_spider"): - self.methods["open_spider"].append(pipe.open_spider) - self._check_mw_method_spider_arg(pipe.open_spider) - if hasattr(pipe, "close_spider"): - self.methods["close_spider"].appendleft(pipe.close_spider) - self._check_mw_method_spider_arg(pipe.close_spider) - if hasattr(pipe, "process_item"): - self.methods["process_item"].append(pipe.process_item) - self._check_mw_method_spider_arg(pipe.process_item) + def _add_middleware(self, mw: Any) -> None: + if hasattr(mw, "open_spider"): + self.methods["open_spider"].append(mw.open_spider) + self._check_mw_method_spider_arg(mw.open_spider) + if hasattr(mw, "close_spider"): + self.methods["close_spider"].appendleft(mw.close_spider) + self._check_mw_method_spider_arg(mw.close_spider) + if hasattr(mw, "process_item"): + self.methods["process_item"].append(mw.process_item) + self._check_mw_method_spider_arg(mw.process_item) def process_item(self, item: Any, spider: Spider) -> Deferred[Any]: warnings.warn( @@ -62,32 +62,44 @@ class ItemPipelineManager(MiddlewareManager): "process_item", item, add_spider=True, warn_deferred=True ) + def _get_dfd( + self, + method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None], + ) -> Deferred[None]: + if method in self._mw_methods_requiring_spider: + return _maybeDeferred_coro(method, True, self._spider) + return _maybeDeferred_coro(method, True) + + @staticmethod + def _eb(failure: Failure) -> Failure: + assert isinstance(failure.value, FirstError) + return failure.value.subFailure + def _process_parallel_dfd(self, methodname: str) -> Deferred[list[None]]: methods = cast( "Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]", self.methods[methodname], ) - def get_dfd( - method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None], - ) -> Deferred[None]: - if method in self._mw_methods_requiring_spider: - return _maybeDeferred_coro(method, True, self._spider) - return _maybeDeferred_coro(method, True) - - dfds = [get_dfd(m) for m in methods] + dfds = [self._get_dfd(m) for m in methods] d: Deferred[list[tuple[bool, None]]] = DeferredList( dfds, fireOnOneErrback=True, consumeErrors=True ) d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r]) - def eb(failure: Failure) -> Failure: - assert isinstance(failure.value, FirstError) - return failure.value.subFailure - - d2.addErrback(eb) + d2.addErrback(self._eb) return d2 + def get_awaitable( + self, + method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None], + ) -> Awaitable[None]: + if method in self._mw_methods_requiring_spider: + result = method(self._spider) + else: + result = method() + return ensure_awaitable(result, _warn=global_object_name(method)) + async def _process_parallel_asyncio(self, methodname: str) -> list[None]: methods = cast( "Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]", @@ -96,16 +108,7 @@ class ItemPipelineManager(MiddlewareManager): if not methods: return [] - def get_awaitable( - method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None], - ) -> Awaitable[None]: - if method in self._mw_methods_requiring_spider: - result = method(self._spider) - else: - result = method() - return ensure_awaitable(result, _warn=global_object_name(method)) - - awaitables = [get_awaitable(m) for m in methods] + awaitables = [self.get_awaitable(m) for m in methods] await asyncio.gather(*awaitables) return [None for _ in methods] diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 1eb17c3d3..ffed96446 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -186,16 +186,18 @@ class S3FilesStore: raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'") self.bucket, self.prefix = uri[5:].split("/", 1) + @staticmethod + def _onsuccess(boto_key: dict[str, Any]) -> StatInfo: + checksum = boto_key["ETag"].strip('"') + last_modified = boto_key["LastModified"] + modified_stamp = time.mktime(last_modified.timetuple()) + return {"checksum": checksum, "last_modified": modified_stamp} + def stat_file( self, path: str, info: MediaPipeline.SpiderInfo ) -> Deferred[StatInfo]: - def _onsuccess(boto_key: dict[str, Any]) -> StatInfo: - checksum = boto_key["ETag"].strip('"') - last_modified = boto_key["LastModified"] - modified_stamp = time.mktime(last_modified.timetuple()) - return {"checksum": checksum, "last_modified": modified_stamp} - return self._get_boto_key(path).addCallback(_onsuccess) + return self._get_boto_key(path).addCallback(self._onsuccess) def _get_boto_key(self, path: str) -> Deferred[dict[str, Any]]: key_name = f"{self.prefix}{path}" @@ -308,20 +310,22 @@ class GCSFilesStore: {"bucket": bucket}, ) + @staticmethod + def _onsuccess(blob: Any) -> StatInfo: + if blob: + checksum = base64.b64decode(blob.md5_hash).hex() + last_modified = time.mktime(blob.updated.timetuple()) + return {"checksum": checksum, "last_modified": last_modified} + return {} + def stat_file( self, path: str, info: MediaPipeline.SpiderInfo ) -> Deferred[StatInfo]: - def _onsuccess(blob: Any) -> StatInfo: - if blob: - checksum = base64.b64decode(blob.md5_hash).hex() - last_modified = time.mktime(blob.updated.timetuple()) - return {"checksum": checksum, "last_modified": last_modified} - return {} blob_path = self._get_blob_path(path) return deferred_from_coro( run_in_thread(self.bucket.get_blob, blob_path) - ).addCallback(_onsuccess) + ).addCallback(self._onsuccess) def _get_content_type(self, headers: dict[str, str] | None) -> str: if headers and "Content-Type" in headers: @@ -395,26 +399,26 @@ class FTPFilesStore: ) ) + def _stat_file(self, path: str) -> StatInfo: + try: + with FTP() as ftp: + ftp.connect(self.host, self.port) + ftp.login(self.username, self.password) + if self.USE_ACTIVE_MODE: + ftp.set_pasv(False) + file_path = f"{self.basedir}/{path}" + last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) + m = hashlib.md5() # noqa: S324 + ftp.retrbinary(f"RETR {file_path}", m.update) + return {"last_modified": last_modified, "checksum": m.hexdigest()} + # The file doesn't exist + except Exception: + return {} + def stat_file( self, path: str, info: MediaPipeline.SpiderInfo ) -> Deferred[StatInfo]: - def _stat_file(path: str) -> StatInfo: - try: - with FTP() as ftp: - ftp.connect(self.host, self.port) - ftp.login(self.username, self.password) - if self.USE_ACTIVE_MODE: - ftp.set_pasv(False) - file_path = f"{self.basedir}/{path}" - last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip()) - m = hashlib.md5() # noqa: S324 - ftp.retrbinary(f"RETR {file_path}", m.update) - return {"last_modified": last_modified, "checksum": m.hexdigest()} - # The file doesn't exist - except Exception: - return {} - - return deferred_from_coro(run_in_thread(_stat_file, path)) + return deferred_from_coro(run_in_thread(self._stat_file, path)) class FilesPipeline(MediaPipeline): @@ -534,43 +538,51 @@ class FilesPipeline(MediaPipeline): store_cls = self.STORE_SCHEMES[scheme] return store_cls(uri) + def _onsuccess( + self, + result: StatInfo, + request: Request, + info: MediaPipeline.SpiderInfo, + path: str, + ) -> FileInfo | None: + if not result: + return None # returning None force download + + last_modified = result.get("last_modified", None) + if not last_modified: + return None # returning None force download + + age_seconds = time.time() - last_modified + age_days = age_seconds / 60 / 60 / 24 + if age_days > self.expires: + return None # returning None force download + + referer = referer_str(request) + logger.debug( + "File (uptodate): Downloaded %(medianame)s from %(request)s " + "referred in <%(referer)s>", + {"medianame": self.MEDIA_NAME, "request": request, "referer": referer}, + extra={"spider": info.spider}, + ) + self.inc_stats("uptodate") + + checksum = result.get("checksum", None) + return { + "url": request.url, + "path": path, + "checksum": checksum, + "status": "uptodate", + } + def media_to_download( self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None ) -> Deferred[FileInfo | None] | None: - def _onsuccess(result: StatInfo) -> FileInfo | None: - if not result: - return None # returning None force download - - last_modified = result.get("last_modified", None) - if not last_modified: - return None # returning None force download - - age_seconds = time.time() - last_modified - age_days = age_seconds / 60 / 60 / 24 - if age_days > self.expires: - return None # returning None force download - - referer = referer_str(request) - logger.debug( - "File (uptodate): Downloaded %(medianame)s from %(request)s " - "referred in <%(referer)s>", - {"medianame": self.MEDIA_NAME, "request": request, "referer": referer}, - extra={"spider": info.spider}, - ) - self.inc_stats("uptodate") - - checksum = result.get("checksum", None) - return { - "url": request.url, - "path": path, - "checksum": checksum, - "status": "uptodate", - } - path = self.file_path(request, info=info, item=item) # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info) # type: ignore[call-overload] - dfd2: Deferred[FileInfo | None] = dfd.addCallback(_onsuccess) + dfd2: Deferred[FileInfo | None] = dfd.addCallback( + functools.partial(self._onsuccess, request=request, info=info, path=path) + ) dfd2.addErrback(lambda _: None) dfd2.addErrback( lambda f: logger.error( diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index 2a1b1932c..d4671fd35 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -143,7 +143,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).") self.set(name, [v for v in value if v != item], self.getpriority(name) or 0) - def get(self, name: _SettingsKey, default: Any = None) -> Any: + def get(self, name: _SettingsKey, default: Any = None) -> Any: # pylint: disable=arguments-renamed """ Get a setting value without affecting its original type. @@ -510,7 +510,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): component_priority_dict[cls] = priority self.set(name, component_priority_dict, self.getpriority(name) or 0) - def setdefault( + def setdefault( # pylint: disable=arguments-renamed self, name: _SettingsKey, default: Any = None, @@ -691,14 +691,14 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): else: p.text(pformat(self.copy_to_dict())) - def pop(self, name: _SettingsKey, default: Any = __default) -> Any: + def pop(self, name: _SettingsKey, default: Any = __default) -> Any: # pylint: disable=arguments-renamed try: value = self.attributes[name].value except KeyError: if default is self.__default: raise return default - self.__delitem__(name) + del self[name] return value diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py index 59f04b8ad..c4c96ef00 100644 --- a/scrapy/utils/benchserver.py +++ b/scrapy/utils/benchserver.py @@ -9,7 +9,7 @@ from twisted.web.server import Request, Site class Root(Resource): isLeaf = True - def getChild(self, name: str, request: Request) -> Resource: + def getChild(self, path: str, request: Request) -> Resource: return self def render(self, request: Request) -> bytes: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 7d7636fe6..90fd04777 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -262,7 +262,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]): def _call_anext(self) -> None: # This starts waiting for the next result from aiterator. # If aiterator is exhausted, _errback will be called. - self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) + self.anext_deferred = deferred_from_coro(anext(self.aiterator)) self.anext_deferred.addCallbacks(self._callback, self._errback) def __next__(self) -> Deferred[Any]: @@ -370,10 +370,10 @@ async def aiter_errback( """Wrap an async iterable calling an errback if an error is caught while iterating it. Similar to :func:`scrapy.utils.defer.iter_errback`. """ - it = aiterable.__aiter__() + it = aiter(aiterable) while True: try: - yield await it.__anext__() + yield await anext(it) except StopAsyncIteration: break except Exception: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index fba80484c..757756e65 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -255,6 +255,11 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]: _generator_callbacks_cache = LocalWeakReferencedCache(limit=128) +def _returns_none(return_node: ast.Return) -> bool: + value = return_node.value + return value is None or (isinstance(value, ast.Constant) and value.value is None) + + def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: # noqa: A002 """ Returns True if a callable is a generator function which includes a @@ -263,12 +268,6 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: # noq if callable in _generator_callbacks_cache: return bool(_generator_callbacks_cache[callable]) - def returns_none(return_node: ast.Return) -> bool: - value = return_node.value - return value is None or ( - isinstance(value, ast.Constant) and value.value is None - ) - if inspect.isgeneratorfunction(callable): func = callable while isinstance(func, partial): @@ -284,7 +283,7 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: # noq tree = ast.parse(code) for node in walk_callable(tree): - if isinstance(node, ast.Return) and not returns_none(node): + if isinstance(node, ast.Return) and not _returns_none(node): _generator_callbacks_cache[callable] = True return bool(_generator_callbacks_cache[callable]) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index bf70fa7fc..9cb695111 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -99,6 +99,16 @@ def to_bytes( return text.encode(encoding, errors) +def _chunk_iter(text: str, chunk_size: int) -> Iterable[tuple[str, int]]: + offset = len(text) + while True: + offset -= chunk_size * 1024 + if offset <= 0: + break + yield (text[offset:], offset) + yield (text, 0) + + def re_rsearch( pattern: str | Pattern[str], text: str, chunk_size: int = 1024 ) -> tuple[int, int] | None: @@ -115,19 +125,10 @@ def re_rsearch( the start position of the match, and the ending (regarding the entire text). """ - def _chunk_iter() -> Iterable[tuple[str, int]]: - offset = len(text) - while True: - offset -= chunk_size * 1024 - if offset <= 0: - break - yield (text[offset:], offset) - yield (text, 0) - if isinstance(pattern, str): pattern = re.compile(pattern) - for chunk, offset in _chunk_iter(): + for chunk, offset in _chunk_iter(text, chunk_size): matches = list(pattern.finditer(chunk)) if matches: start, end = matches[-1].span() diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py index 6cf0046c0..622324a10 100644 --- a/tests/mockserver/http.py +++ b/tests/mockserver/http.py @@ -85,7 +85,7 @@ class Root(resource.Resource): self.putChild(b"response-headers", ResponseHeadersResource()) self.putChild(b"set-cookie", SetCookie()) - def getChild(self, name, request): + def getChild(self, path, request): return self def render(self, request): diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.py index 5f23dd2c4..a8f483ee1 100644 --- a/tests/mockserver/simple_https.py +++ b/tests/mockserver/simple_https.py @@ -13,7 +13,7 @@ class Root(resource.Resource): resource.Resource.__init__(self) self.putChild(b"file", Data(b"0123456789", "text/plain")) - def getChild(self, name, request): + def getChild(self, path, request): return self diff --git a/tox.ini b/tox.ini index f1b004ab8..07583a11e 100644 --- a/tox.ini +++ b/tox.ini @@ -89,8 +89,9 @@ basepython = python3 deps = {[testenv:extra-deps]deps} pylint==4.0.2 + pylint-per-file-ignores # https://github.com/pylint-dev/pylint/issues/3767#issuecomment-1319916278 commands = - pylint conftest.py docs extras scrapy tests + pylint {posargs:conftest.py docs extras scrapy tests} [testenv:twinecheck] basepython = python3