diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 3c7960b86..4b4d4efc7 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -21,6 +21,7 @@ from scrapy.utils.asyncio import ( call_later, create_looping_call, ) +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.defer import ( _defer_sleep_async, _schedule_coro, @@ -144,15 +145,10 @@ class Downloader: ) @inlineCallbacks + @_warn_spider_arg def fetch( self, request: Request, spider: Spider | None = None ) -> Generator[Deferred[Any], Any, Response | Request]: - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to Downloader.fetch() is deprecated.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) self.active.add(request) try: return (yield self.middleware.download(self._enqueue_request, request)) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index fe3c780e3..6352ede3c 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -3,13 +3,13 @@ from __future__ import annotations import logging -import warnings from typing import TYPE_CHECKING, Any, Protocol, cast from twisted.internet import defer from scrapy import Request, Spider, signals -from scrapy.exceptions import NotConfigured, NotSupported, ScrapyDeprecationWarning +from scrapy.exceptions import NotConfigured, NotSupported +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import without_none_values @@ -94,15 +94,10 @@ class DownloadHandlers: self._handlers[scheme] = dh return dh + @_warn_spider_arg def download_request( self, request: Request, spider: Spider | None = None ) -> Deferred[Response]: - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to DownloadHandlers.download_request() is deprecated.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) scheme = urlparse_cached(request).scheme handler = self._get_handler(scheme) if not handler: diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 769b32fc9..9184731cd 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -23,6 +23,7 @@ from scrapy.exceptions import ( from scrapy.http import Request, Response from scrapy.pipelines import ItemPipelineManager from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.defer import ( _defer_sleep_async, _schedule_coro, @@ -219,16 +220,10 @@ class Scraper: self.slot.closing.callback(self.crawler.spider) @inlineCallbacks + @_warn_spider_arg def enqueue_scrape( self, result: Response | Failure, request: Request, spider: Spider | None = None ) -> Generator[Deferred[Any], Any, None]: - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to Scraper.enqueue_scrape() is deprecated.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - if self.slot is None: raise RuntimeError("Scraper slot not assigned") dfd = self.slot.add_response_request(result, request) @@ -343,6 +338,7 @@ class Scraper: # which needs to be passed to iterate_spider_output() return await ensure_awaitable(iterate_spider_output(output)) + @_warn_spider_arg def handle_spider_error( self, _failure: Failure, @@ -351,13 +347,6 @@ class Scraper: spider: Spider | None = None, ) -> None: """Handle an exception raised by a spider callback or errback.""" - if spider is not None: - warnings.warn( - "Passing a 'spider' argument to Scraper.handle_spider_error() is deprecated.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - assert self.crawler.spider exc = _failure.value if isinstance(exc, CloseSpider): diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index fb65ff06a..a7cd51166 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -22,6 +22,7 @@ from scrapy.http.request import NO_CALLBACK, Request from scrapy.settings import Settings from scrapy.utils.asyncio import call_later from scrapy.utils.datatypes import SequenceExclude +from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.defer import _DEFER_DELAY, _defer_sleep, deferred_from_coro from scrapy.utils.log import failure_to_exc_info from scrapy.utils.misc import arg_to_iter @@ -165,27 +166,15 @@ class MediaPipeline(ABC): pipe._finish_init(crawler) return pipe + @_warn_spider_arg def open_spider(self, spider: Spider | None = None) -> None: - if spider is not None: # pragma: no cover - warnings.warn( - "Passing a spider argument to MediaPipeline.open_spider()" - " is deprecated and the passed value is ignored.", - ScrapyDeprecationWarning, - stacklevel=2, - ) assert self.crawler.spider self.spiderinfo = self.SpiderInfo(self.crawler.spider) + @_warn_spider_arg def process_item( self, item: Any, spider: Spider | None = None ) -> Deferred[list[FileInfoOrError]]: - if spider is not None: # pragma: no cover - warnings.warn( - "Passing a spider argument to MediaPipeline.process_item()" - " is deprecated and the passed value is ignored.", - ScrapyDeprecationWarning, - stacklevel=2, - ) info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) dlist = [self._process_request(r, info, item) for r in requests] diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index a047809c3..889fc6df1 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -1,10 +1,9 @@ from __future__ import annotations -import warnings from typing import TYPE_CHECKING, Any from scrapy import Request, Spider -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.decorators import _warn_spider_arg if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable @@ -54,33 +53,21 @@ class BaseSpiderMiddleware: if (o := self._get_processed(o, None)) is not None: yield o + @_warn_spider_arg def process_spider_output( self, response: Response, result: Iterable[Any], spider: Spider | None = None ) -> Iterable[Any]: - if spider is not None: # pragma: no cover - warnings.warn( - "Passing a spider argument to BaseSpiderMiddleware.process_spider_output()" - " is deprecated and the passed value is ignored.", - ScrapyDeprecationWarning, - stacklevel=2, - ) for o in result: if (o := self._get_processed(o, response)) is not None: yield o + @_warn_spider_arg async def process_spider_output_async( self, response: Response, result: AsyncIterator[Any], spider: Spider | None = None, ) -> AsyncIterator[Any]: - if spider is not None: # pragma: no cover - warnings.warn( - "Passing a spider argument to BaseSpiderMiddleware.process_spider_output_async()" - " is deprecated and the passed value is ignored.", - ScrapyDeprecationWarning, - stacklevel=2, - ) async for o in result: if (o := self._get_processed(o, response)) is not None: yield o diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index d23bca50c..9c387530c 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -7,11 +7,10 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -import warnings from typing import TYPE_CHECKING, Any -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.spidermiddlewares.base import BaseSpiderMiddleware +from scrapy.utils.decorators import _warn_spider_arg if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable @@ -54,32 +53,20 @@ class DepthMiddleware(BaseSpiderMiddleware): o.crawler = crawler return o + @_warn_spider_arg def process_spider_output( self, response: Response, result: Iterable[Any], spider: Spider | None = None ) -> Iterable[Any]: - if spider is not None: # pragma: no cover - warnings.warn( - "Passing a spider argument to DepthMiddleware.process_spider_output()" - " is deprecated and the passed value is ignored.", - ScrapyDeprecationWarning, - stacklevel=2, - ) self._init_depth(response) yield from super().process_spider_output(response, result) + @_warn_spider_arg async def process_spider_output_async( self, response: Response, result: AsyncIterator[Any], spider: Spider | None = None, ) -> AsyncIterator[Any]: - if spider is not None: # pragma: no cover - warnings.warn( - "Passing a spider argument to DepthMiddleware.process_spider_output_async()" - " is deprecated and the passed value is ignored.", - ScrapyDeprecationWarning, - stacklevel=2, - ) self._init_depth(response) async for o in super().process_spider_output_async(response, result): yield o diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index b4c9c10bf..c160d1adb 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -7,10 +7,10 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -import warnings from typing import TYPE_CHECKING, Any -from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning +from scrapy.exceptions import IgnoreRequest +from scrapy.utils.decorators import _warn_spider_arg if TYPE_CHECKING: from collections.abc import Iterable @@ -50,16 +50,10 @@ class HttpErrorMiddleware: o.crawler = crawler return o + @_warn_spider_arg def process_spider_input( self, response: Response, spider: Spider | None = None ) -> None: - if spider is not None: # pragma: no cover - warnings.warn( - "Passing a spider argument to HttpErrorMiddleware.process_spider_input()" - " is deprecated and the passed value is ignored.", - ScrapyDeprecationWarning, - stacklevel=2, - ) if 200 <= response.status < 300: # common case return meta = response.meta @@ -79,16 +73,10 @@ class HttpErrorMiddleware: return raise HttpError(response, "Ignoring non-200 response") + @_warn_spider_arg def process_spider_exception( self, response: Response, exception: Exception, spider: Spider | None = None ) -> Iterable[Any] | None: - if spider is not None: # pragma: no cover - warnings.warn( - "Passing a spider argument to HttpErrorMiddleware.process_spider_exception()" - " is deprecated and the passed value is ignored.", - ScrapyDeprecationWarning, - stacklevel=2, - ) if isinstance(exception, HttpError): assert self.crawler.stats self.crawler.stats.inc_value("httperror/response_ignored_count") diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py index c2fa0ae6a..02adef1e4 100644 --- a/scrapy/statscollectors.py +++ b/scrapy/statscollectors.py @@ -4,13 +4,11 @@ Scrapy extension for collecting scraping stats from __future__ import annotations -import inspect import logging import pprint -import warnings from typing import TYPE_CHECKING, Any -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.decorators import _warn_spider_arg if TYPE_CHECKING: from scrapy import Spider @@ -44,22 +42,7 @@ class StatsCollector: "open_spider", "close_spider", ) and callable(original_attr): - - def _deprecated_wrapper(*args, **kwargs): - sig = inspect.signature(original_attr).bind(*args, **kwargs) - sig.apply_defaults() - - if sig.arguments.get("spider"): - warnings.warn( - f"Passing a 'spider' argument to StatsCollector.{name}() is deprecated and" - f" the argument will be removed in a future Scrapy version.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - - return original_attr(*args, **kwargs) - - return _deprecated_wrapper + return _warn_spider_arg(original_attr) return original_attr diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py index 0f4d0beda..c4739bcee 100644 --- a/scrapy/utils/decorators.py +++ b/scrapy/utils/decorators.py @@ -1,8 +1,9 @@ from __future__ import annotations +import inspect import warnings from functools import wraps -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar, overload from twisted.internet.defer import Deferred, maybeDeferred from twisted.internet.threads import deferToThread @@ -10,7 +11,7 @@ from twisted.internet.threads import deferToThread from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: - from collections.abc import Callable + from collections.abc import AsyncGenerator, Callable, Coroutine # typing.ParamSpec requires Python 3.10 from typing_extensions import ParamSpec @@ -30,7 +31,7 @@ def deprecated( def deco(func: Callable[_P, _T]) -> Callable[_P, _T]: @wraps(func) - def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> Any: + def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _T: message = f"Call to deprecated function {func.__name__}." if use_instead: message += f" Use {use_instead} instead." @@ -65,3 +66,67 @@ def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]: return deferToThread(func, *a, **kw) return wrapped + + +@overload +def _warn_spider_arg( + func: Callable[_P, Coroutine[Any, Any, _T]], +) -> Callable[_P, Coroutine[Any, Any, _T]]: ... + + +@overload +def _warn_spider_arg( + func: Callable[_P, AsyncGenerator[_T]], +) -> Callable[_P, AsyncGenerator[_T]]: ... + + +@overload +def _warn_spider_arg(func: Callable[_P, _T]) -> Callable[_P, _T]: ... + + +def _warn_spider_arg( + func: Callable[_P, _T], +) -> ( + Callable[_P, _T] + | Callable[_P, Coroutine[Any, Any, _T]] + | Callable[_P, AsyncGenerator[_T]] +): + """Decorator to warn if a ``spider`` argument is passed to a function.""" + + def check_args(*args: _P.args, **kwargs: _P.kwargs) -> None: + bound = inspect.signature(func).bind(*args, **kwargs) + if "spider" in bound.arguments: + warnings.warn( + f"Passing a 'spider' argument to {func.__qualname__}() is deprecated and " + "the argument will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=3, + ) + + if inspect.iscoroutinefunction(func): + + @wraps(func) + async def async_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T: + check_args(*args, **kwargs) + return await func(*args, **kwargs) + + return async_inner + + if inspect.isasyncgenfunction(func): + + @wraps(func) + async def asyncgen_inner( + *args: _P.args, **kwargs: _P.kwargs + ) -> AsyncGenerator[_T]: + check_args(*args, **kwargs) + async for item in func(*args, **kwargs): + yield item + + return asyncgen_inner + + @wraps(func) + def sync_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T: + check_args(*args, **kwargs) + return func(*args, **kwargs) + + return sync_inner diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 5276636fc..99d5cbb6d 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -180,7 +180,7 @@ class TestFilesPipeline: for p in patchers: p.start() - result = yield self.pipeline.process_item(item, None) + result = yield self.pipeline.process_item(item) assert result["files"][0]["checksum"] == "abc" assert result["files"][0]["status"] == "uptodate" @@ -211,7 +211,7 @@ class TestFilesPipeline: for p in patchers: p.start() - result = yield self.pipeline.process_item(item, None) + result = yield self.pipeline.process_item(item) assert result["files"][0]["checksum"] != "abc" assert result["files"][0]["status"] == "downloaded" @@ -242,7 +242,7 @@ class TestFilesPipeline: for p in patchers: p.start() - result = yield self.pipeline.process_item(item, None) + result = yield self.pipeline.process_item(item) assert result["files"][0]["checksum"] != "abc" assert result["files"][0]["status"] == "cached" diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py index e2ee48e1d..da570904d 100644 --- a/tests/test_spidermiddleware_base.py +++ b/tests/test_spidermiddleware_base.py @@ -28,7 +28,7 @@ def test_trivial(crawler: Crawler) -> None: test_req = Request("data:,") spider_output = [test_req, {"foo": "bar"}] for processed in [ - list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type] + list(mw.process_spider_output(Response("data:,"), spider_output)), list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type] ]: assert processed == [test_req, {"foo": "bar"}] @@ -51,7 +51,7 @@ def test_processed_request(crawler: Crawler) -> None: test_req3 = Request("data:3,") spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3] for processed in [ - list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type] + list(mw.process_spider_output(Response("data:,"), spider_output)), list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type] ]: assert len(processed) == 3 @@ -75,7 +75,7 @@ def test_processed_item(crawler: Crawler) -> None: test_req = Request("data:,") spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}] for processed in [ - list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type] + list(mw.process_spider_output(Response("data:,"), spider_output)), list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type] ]: assert processed == [{"foo": 1}, test_req, {"foo": 30}] @@ -112,7 +112,7 @@ def test_processed_both(crawler: Crawler) -> None: test_req3, ] for processed in [ - list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type] + list(mw.process_spider_output(Response("data:,"), spider_output)), list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type] ]: assert len(processed) == 4