RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Extract @_warn_spider_arg. (#7033) 

* Extract @_warn_spider_arg.

* Also deprecate passing spider=None.
This commit is contained in:
Andrey Rakhmatullin 2025-09-01 19:17:13 +04:00 committed by GitHub
parent 80beec41b5
commit c097921c44
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 98 additions and 119 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/core/downloader/__init__.py
View File

@ -21,6 +21,7 @@ from scrapy.utils.asyncio import (
call_later,
create_looping_call,
)
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import (
_defer_sleep_async,
_schedule_coro,
@ -144,15 +145,10 @@ class Downloader:
)
@inlineCallbacks
@_warn_spider_arg
def fetch(
self, request: Request, spider: Spider | None = None
) -> Generator[Deferred[Any], Any, Response | Request]:
if spider is not None:
warnings.warn(
"Passing a 'spider' argument to Downloader.fetch() is deprecated.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
self.active.add(request)
try:
return (yield self.middleware.download(self._enqueue_request, request))

11
scrapy/core/downloader/handlers/__init__.py
View File

@ -3,13 +3,13 @@
from __future__ import annotations
import logging
import warnings
from typing import TYPE_CHECKING, Any, Protocol, cast
from twisted.internet import defer
from scrapy import Request, Spider, signals
from scrapy.exceptions import NotConfigured, NotSupported, ScrapyDeprecationWarning
from scrapy.exceptions import NotConfigured, NotSupported
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import without_none_values
@ -94,15 +94,10 @@ class DownloadHandlers:
self._handlers[scheme] = dh
return dh
@_warn_spider_arg
def download_request(
self, request: Request, spider: Spider | None = None
) -> Deferred[Response]:
if spider is not None:
warnings.warn(
"Passing a 'spider' argument to DownloadHandlers.download_request() is deprecated.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
scheme = urlparse_cached(request).scheme
handler = self._get_handler(scheme)
if not handler:

17
scrapy/core/scraper.py
View File

@ -23,6 +23,7 @@ from scrapy.exceptions import (
from scrapy.http import Request, Response
from scrapy.pipelines import ItemPipelineManager
from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import (
_defer_sleep_async,
_schedule_coro,
@ -219,16 +220,10 @@ class Scraper:
self.slot.closing.callback(self.crawler.spider)
@inlineCallbacks
@_warn_spider_arg
def enqueue_scrape(
self, result: Response | Failure, request: Request, spider: Spider | None = None
) -> Generator[Deferred[Any], Any, None]:
if spider is not None:
warnings.warn(
"Passing a 'spider' argument to Scraper.enqueue_scrape() is deprecated.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
if self.slot is None:
raise RuntimeError("Scraper slot not assigned")
dfd = self.slot.add_response_request(result, request)
@ -343,6 +338,7 @@ class Scraper:
# which needs to be passed to iterate_spider_output()
return await ensure_awaitable(iterate_spider_output(output))
@_warn_spider_arg
def handle_spider_error(
self,
_failure: Failure,
@ -351,13 +347,6 @@ class Scraper:
spider: Spider | None = None,
) -> None:
"""Handle an exception raised by a spider callback or errback."""
if spider is not None:
warnings.warn(
"Passing a 'spider' argument to Scraper.handle_spider_error() is deprecated.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
assert self.crawler.spider
exc = _failure.value
if isinstance(exc, CloseSpider):

17
scrapy/pipelines/media.py
View File

@ -22,6 +22,7 @@ from scrapy.http.request import NO_CALLBACK, Request
from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import _DEFER_DELAY, _defer_sleep, deferred_from_coro
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import arg_to_iter
@ -165,27 +166,15 @@ class MediaPipeline(ABC):
pipe._finish_init(crawler)
return pipe
@_warn_spider_arg
def open_spider(self, spider: Spider | None = None) -> None:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to MediaPipeline.open_spider()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
assert self.crawler.spider
self.spiderinfo = self.SpiderInfo(self.crawler.spider)
@_warn_spider_arg
def process_item(
self, item: Any, spider: Spider | None = None
) -> Deferred[list[FileInfoOrError]]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to MediaPipeline.process_item()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
info = self.spiderinfo
requests = arg_to_iter(self.get_media_requests(item, info))
dlist = [self._process_request(r, info, item) for r in requests]

19
scrapy/spidermiddlewares/base.py
View File

@ -1,10 +1,9 @@
from __future__ import annotations
import warnings
from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterable
@ -54,33 +53,21 @@ class BaseSpiderMiddleware:
if (o := self._get_processed(o, None)) is not None:
yield o
@_warn_spider_arg
def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider | None = None
) -> Iterable[Any]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to BaseSpiderMiddleware.process_spider_output()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
for o in result:
if (o := self._get_processed(o, response)) is not None:
yield o
@_warn_spider_arg
async def process_spider_output_async(
self,
response: Response,
result: AsyncIterator[Any],
spider: Spider | None = None,
) -> AsyncIterator[Any]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to BaseSpiderMiddleware.process_spider_output_async()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
async for o in result:
if (o := self._get_processed(o, response)) is not None:
yield o

19
scrapy/spidermiddlewares/depth.py
View File

@ -7,11 +7,10 @@ See documentation in docs/topics/spider-middleware.rst
from __future__ import annotations
import logging
import warnings
from typing import TYPE_CHECKING, Any
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterable
@ -54,32 +53,20 @@ class DepthMiddleware(BaseSpiderMiddleware):
o.crawler = crawler
return o
@_warn_spider_arg
def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider | None = None
) -> Iterable[Any]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to DepthMiddleware.process_spider_output()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
self._init_depth(response)
yield from super().process_spider_output(response, result)
@_warn_spider_arg
async def process_spider_output_async(
self,
response: Response,
result: AsyncIterator[Any],
spider: Spider | None = None,
) -> AsyncIterator[Any]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to DepthMiddleware.process_spider_output_async()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
self._init_depth(response)
async for o in super().process_spider_output_async(response, result):
yield o

20
scrapy/spidermiddlewares/httperror.py
View File

@ -7,10 +7,10 @@ See documentation in docs/topics/spider-middleware.rst
from __future__ import annotations
import logging
import warnings
from typing import TYPE_CHECKING, Any
from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING:
from collections.abc import Iterable
@ -50,16 +50,10 @@ class HttpErrorMiddleware:
o.crawler = crawler
return o
@_warn_spider_arg
def process_spider_input(
self, response: Response, spider: Spider | None = None
) -> None:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to HttpErrorMiddleware.process_spider_input()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
if 200 <= response.status < 300: # common case
return
meta = response.meta
@ -79,16 +73,10 @@ class HttpErrorMiddleware:
return
raise HttpError(response, "Ignoring non-200 response")
@_warn_spider_arg
def process_spider_exception(
self, response: Response, exception: Exception, spider: Spider | None = None
) -> Iterable[Any] | None:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to HttpErrorMiddleware.process_spider_exception()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
if isinstance(exception, HttpError):
assert self.crawler.stats
self.crawler.stats.inc_value("httperror/response_ignored_count")

21
scrapy/statscollectors.py
View File

@ -4,13 +4,11 @@ Scrapy extension for collecting scraping stats
from __future__ import annotations
import inspect
import logging
import pprint
import warnings
from typing import TYPE_CHECKING, Any
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.decorators import _warn_spider_arg
if TYPE_CHECKING:
from scrapy import Spider
@ -44,22 +42,7 @@ class StatsCollector:
"open_spider",
"close_spider",
) and callable(original_attr):
def _deprecated_wrapper(*args, **kwargs):
sig = inspect.signature(original_attr).bind(*args, **kwargs)
sig.apply_defaults()
if sig.arguments.get("spider"):
warnings.warn(
f"Passing a 'spider' argument to StatsCollector.{name}() is deprecated and"
f" the argument will be removed in a future Scrapy version.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return original_attr(*args, **kwargs)
return _deprecated_wrapper
return _warn_spider_arg(original_attr)
return original_attr

71
scrapy/utils/decorators.py
View File

@ -1,8 +1,9 @@
from __future__ import annotations
import inspect
import warnings
from functools import wraps
from typing import TYPE_CHECKING, Any, TypeVar
from typing import TYPE_CHECKING, Any, TypeVar, overload
from twisted.internet.defer import Deferred, maybeDeferred
from twisted.internet.threads import deferToThread
@ -10,7 +11,7 @@ from twisted.internet.threads import deferToThread
from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING:
from collections.abc import Callable
from collections.abc import AsyncGenerator, Callable, Coroutine
# typing.ParamSpec requires Python 3.10
from typing_extensions import ParamSpec
@ -30,7 +31,7 @@ def deprecated(
def deco(func: Callable[_P, _T]) -> Callable[_P, _T]:
@wraps(func)
def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> Any:
def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _T:
message = f"Call to deprecated function {func.__name__}."
if use_instead:
message += f" Use {use_instead} instead."
@ -65,3 +66,67 @@ def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]:
return deferToThread(func, *a, **kw)
return wrapped
@overload
def _warn_spider_arg(
func: Callable[_P, Coroutine[Any, Any, _T]],
) -> Callable[_P, Coroutine[Any, Any, _T]]: ...
@overload
def _warn_spider_arg(
func: Callable[_P, AsyncGenerator[_T]],
) -> Callable[_P, AsyncGenerator[_T]]: ...
@overload
def _warn_spider_arg(func: Callable[_P, _T]) -> Callable[_P, _T]: ...
def _warn_spider_arg(
func: Callable[_P, _T],
) -> (
Callable[_P, _T]
| Callable[_P, Coroutine[Any, Any, _T]]
| Callable[_P, AsyncGenerator[_T]]
):
"""Decorator to warn if a ``spider`` argument is passed to a function."""
def check_args(*args: _P.args, **kwargs: _P.kwargs) -> None:
bound = inspect.signature(func).bind(*args, **kwargs)
if "spider" in bound.arguments:
warnings.warn(
f"Passing a 'spider' argument to {func.__qualname__}() is deprecated and "
"the argument will be removed in a future Scrapy version.",
category=ScrapyDeprecationWarning,
stacklevel=3,
)
if inspect.iscoroutinefunction(func):
@wraps(func)
async def async_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T:
check_args(*args, **kwargs)
return await func(*args, **kwargs)
return async_inner
if inspect.isasyncgenfunction(func):
@wraps(func)
async def asyncgen_inner(
*args: _P.args, **kwargs: _P.kwargs
) -> AsyncGenerator[_T]:
check_args(*args, **kwargs)
async for item in func(*args, **kwargs):
yield item
return asyncgen_inner
@wraps(func)
def sync_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T:
check_args(*args, **kwargs)
return func(*args, **kwargs)
return sync_inner

6
tests/test_pipeline_files.py
View File

@ -180,7 +180,7 @@ class TestFilesPipeline:
for p in patchers:
p.start()
result = yield self.pipeline.process_item(item, None)
result = yield self.pipeline.process_item(item)
assert result["files"][0]["checksum"] == "abc"
assert result["files"][0]["status"] == "uptodate"
@ -211,7 +211,7 @@ class TestFilesPipeline:
for p in patchers:
p.start()
result = yield self.pipeline.process_item(item, None)
result = yield self.pipeline.process_item(item)
assert result["files"][0]["checksum"] != "abc"
assert result["files"][0]["status"] == "downloaded"
@ -242,7 +242,7 @@ class TestFilesPipeline:
for p in patchers:
p.start()
result = yield self.pipeline.process_item(item, None)
result = yield self.pipeline.process_item(item)
assert result["files"][0]["checksum"] != "abc"
assert result["files"][0]["status"] == "cached"

8
tests/test_spidermiddleware_base.py
View File

@ -28,7 +28,7 @@ def test_trivial(crawler: Crawler) -> None:
test_req = Request("data:,")
spider_output = [test_req, {"foo": "bar"}]
for processed in [
list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type]
list(mw.process_spider_output(Response("data:,"), spider_output)),
list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
]:
assert processed == [test_req, {"foo": "bar"}]
@ -51,7 +51,7 @@ def test_processed_request(crawler: Crawler) -> None:
test_req3 = Request("data:3,")
spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]
for processed in [
list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type]
list(mw.process_spider_output(Response("data:,"), spider_output)),
list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
]:
assert len(processed) == 3
@ -75,7 +75,7 @@ def test_processed_item(crawler: Crawler) -> None:
test_req = Request("data:,")
spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]
for processed in [
list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type]
list(mw.process_spider_output(Response("data:,"), spider_output)),
list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
]:
assert processed == [{"foo": 1}, test_req, {"foo": 30}]
@ -112,7 +112,7 @@ def test_processed_both(crawler: Crawler) -> None:
test_req3,
]
for processed in [
list(mw.process_spider_output(Response("data:,"), spider_output, None)), # type: ignore[arg-type]
list(mw.process_spider_output(Response("data:,"), spider_output)),
list(mw.process_start_requests(spider_output, None)), # type: ignore[arg-type]
]:
assert len(processed) == 4