diff --git a/docs/faq.rst b/docs/faq.rst index 1d09a0e63..fdf0582de 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -349,7 +349,7 @@ method for this purpose. For example: class MultiplyItemsMiddleware: - def process_spider_output(self, response, result, spider): + def process_spider_output(self, response, result): for item_or_request in result: if isinstance(item_or_request, Request): continue diff --git a/docs/news.rst b/docs/news.rst index 92320ac6a..6a019e2fb 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -44,6 +44,20 @@ Backward-incompatible changes - :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer` +- :class:`scrapy.middleware.MiddlewareManager` no longer includes code for + handling ``open_spider()`` and ``close_spider()`` component methods. As + this code was only used for pipelines it was moved into + :class:`scrapy.pipelines.ItemPipelineManager`. This change should only + affect custom subclasses of :class:`~scrapy.middleware.MiddlewareManager`. + The following code was moved: + + - ``scrapy.middleware.MiddlewareManager.open_spider()`` + + - ``scrapy.middleware.MiddlewareManager.close_spider()`` + + - Code in ``scrapy.middleware.MiddlewareManager._add_middleware()`` that + processes ``open_spider()`` and ``close_spider()`` component methods. + - :meth:`scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware.process_request` now returns a coroutine, previously it returned a :class:`~twisted.internet.defer.Deferred` object or ``None``. The diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index b9d780528..fdd3dcd87 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -191,7 +191,7 @@ shorter and cleaner: adapter["field"] = data return item - def process_item(self, item, spider): + def process_item(self, item): adapter = ItemAdapter(item) dfd = db.get_some_data(adapter["id"]) dfd.addCallback(self._update_item, item) @@ -205,7 +205,7 @@ becomes: class DbPipeline: - async def process_item(self, item, spider): + async def process_item(self, item): adapter = ItemAdapter(item) adapter["field"] = await db.get_some_data(adapter["id"]) return item @@ -421,12 +421,12 @@ For example: .. code-block:: python class UniversalSpiderMiddleware: - def process_spider_output(self, response, result, spider): + def process_spider_output(self, response, result): for r in result: # ... do something with r yield r - async def process_spider_output_async(self, response, result, spider): + async def process_spider_output_async(self, response, result): async for r in result: # ... do something with r yield r diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 2fbe3f75d..d3186ef5c 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -67,7 +67,7 @@ value of one of their fields: self.year_to_exporter[year] = (exporter, xml_file) return self.year_to_exporter[year][0] - def process_item(self, item, spider): + def process_item(self, item): exporter = self._exporter_for_item(item) exporter.export_item(item) return item diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 13a5b88f7..e67cf06c8 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -26,7 +26,7 @@ Writing your own item pipeline Each item pipeline is a :ref:`component ` that must implement the following method: -.. method:: process_item(self, item, spider) +.. method:: process_item(self, item) This method is called for every item pipeline component. @@ -42,25 +42,16 @@ implement the following method: :param item: the scraped item :type item: :ref:`item object ` - :param spider: the spider which scraped the item - :type spider: :class:`~scrapy.Spider` object - Additionally, they may also implement the following methods: -.. method:: open_spider(self, spider) +.. method:: open_spider(self) This method is called when the spider is opened. - :param spider: the spider which was opened - :type spider: :class:`~scrapy.Spider` object - -.. method:: close_spider(self, spider) +.. method:: close_spider(self) This method is called when the spider is closed. - :param spider: the spider which was closed - :type spider: :class:`~scrapy.Spider` object - Item pipeline example ===================== @@ -82,7 +73,7 @@ contain a price: class PricePipeline: vat_factor = 1.15 - def process_item(self, item, spider): + def process_item(self, item): adapter = ItemAdapter(item) if adapter.get("price"): if adapter.get("price_excludes_vat"): @@ -107,13 +98,13 @@ format: class JsonWriterPipeline: - def open_spider(self, spider): + def open_spider(self): self.file = open("items.jsonl", "w") - def close_spider(self, spider): + def close_spider(self): self.file.close() - def process_item(self, item, spider): + def process_item(self, item): line = json.dumps(ItemAdapter(item).asdict()) + "\n" self.file.write(line) return item @@ -153,14 +144,14 @@ The main point of this example is to show how to :ref:`get the crawler mongo_db=crawler.settings.get("MONGO_DATABASE", "items"), ) - def open_spider(self, spider): + def open_spider(self): self.client = pymongo.MongoClient(self.mongo_uri) self.db = self.client[self.mongo_db] - def close_spider(self, spider): + def close_spider(self): self.client.close() - def process_item(self, item, spider): + def process_item(self, item): self.db[self.collection_name].insert_one(ItemAdapter(item).asdict()) return item @@ -198,12 +189,19 @@ item. SPLASH_URL = "http://localhost:8050/render.png?url={}" - async def process_item(self, item, spider): + def __init__(crawler): + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + async def process_item(self, item): adapter = ItemAdapter(item) encoded_item_url = quote(adapter["url"]) screenshot_url = self.SPLASH_URL.format(encoded_item_url) request = scrapy.Request(screenshot_url, callback=NO_CALLBACK) - response = await spider.crawler.engine.download_async(request) + response = await self.crawler.engine.download_async(request) if response.status != 200: # Error happened, return item. @@ -238,7 +236,7 @@ returns multiples items with the same id: def __init__(self): self.ids_seen = set() - def process_item(self, item, spider): + def process_item(self, item): adapter = ItemAdapter(item) if adapter["id"] in self.ids_seen: raise DropItem(f"Item ID already seen: {adapter['id']}") diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 4e9519eef..9dc26d703 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -554,7 +554,7 @@ When writing an item pipeline, you can force a different log level by setting class MyPipeline: - def process_item(self, item, spider): + def process_item(self, item): if not item.get("price"): raise DropItem("Missing price data", log_level="INFO") return item diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2eb59fe44..0f81e409e 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -94,7 +94,7 @@ one or more of these methods: def process_start_requests(self, start, spider): yield from start - .. method:: process_spider_input(response, spider) + .. method:: process_spider_input(response) This method is called for each response that goes through the spider middleware and into the spider, for processing. @@ -116,11 +116,7 @@ one or more of these methods: :param response: the response being processed :type response: :class:`~scrapy.http.Response` object - :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.Spider` object - - - .. method:: process_spider_output(response, result, spider) + .. method:: process_spider_output(response, result) This method is called with the results returned from the Spider, after it has processed the response. @@ -149,10 +145,7 @@ one or more of these methods: :type result: an iterable of :class:`~scrapy.Request` objects and :ref:`item objects ` - :param spider: the spider whose result is being processed - :type spider: :class:`~scrapy.Spider` object - - .. method:: process_spider_output_async(response, result, spider) + .. method:: process_spider_output_async(response, result) :async: .. versionadded:: 2.7 @@ -161,7 +154,7 @@ one or more of these methods: which will be called instead of :meth:`process_spider_output` if ``result`` is an :term:`asynchronous iterable`. - .. method:: process_spider_exception(response, exception, spider) + .. method:: process_spider_exception(response, exception) This method is called when a spider or :meth:`process_spider_output` method (from a previous spider middleware) raises an exception. @@ -186,8 +179,6 @@ one or more of these methods: :param exception: the exception raised :type exception: :exc:`Exception` object - :param spider: the spider which raised the exception - :type spider: :class:`~scrapy.Spider` object Base class for custom spider middlewares ---------------------------------------- diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 9f92e906d..75246710d 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -117,9 +117,10 @@ class SpiderMiddlewareManager(MiddlewareManager): ) def _add_middleware(self, mw: Any) -> None: - super()._add_middleware(mw) if hasattr(mw, "process_spider_input"): self.methods["process_spider_input"].append(mw.process_spider_input) + self._check_mw_method_spider_arg(mw.process_spider_input) + if self._use_start_requests: if hasattr(mw, "process_start_requests"): self.methods["process_start_requests"].appendleft( @@ -127,10 +128,19 @@ class SpiderMiddlewareManager(MiddlewareManager): ) elif hasattr(mw, "process_start"): self.methods["process_start"].appendleft(mw.process_start) + process_spider_output = self._get_async_method_pair(mw, "process_spider_output") self.methods["process_spider_output"].appendleft(process_spider_output) + if callable(process_spider_output): + self._check_mw_method_spider_arg(process_spider_output) + elif isinstance(process_spider_output, tuple): + for m in process_spider_output: + self._check_mw_method_spider_arg(m) + process_spider_exception = getattr(mw, "process_spider_exception", None) self.methods["process_spider_exception"].appendleft(process_spider_exception) + if process_spider_exception is not None: + self._check_mw_method_spider_arg(process_spider_exception) async def _process_spider_input( self, @@ -141,7 +151,10 @@ class SpiderMiddlewareManager(MiddlewareManager): for method in self.methods["process_spider_input"]: method = cast("Callable", method) try: - result = method(response=response, spider=self._spider) + if method in self._mw_methods_requiring_spider: + result = method(response=response, spider=self._spider) + else: + result = method(response=response) if result is not None: msg = ( f"{global_object_name(method)} must return None " @@ -212,7 +225,12 @@ class SpiderMiddlewareManager(MiddlewareManager): if method is None: continue method = cast("Callable", method) - result = method(response=response, exception=exception, spider=self._spider) + if method in self._mw_methods_requiring_spider: + result = method( + response=response, exception=exception, spider=self._spider + ) + else: + result = method(response=response, exception=exception) if _isiterable(result): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned @@ -298,7 +316,12 @@ class SpiderMiddlewareManager(MiddlewareManager): ) recovered = MutableChain(recovered_collected) # might fail directly if the output value is not a generator - result = method(response=response, result=result, spider=self._spider) + if method in self._mw_methods_requiring_spider: + result = method( + response=response, result=result, spider=self._spider + ) + else: + result = method(response=response, result=result) except Exception as ex: exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = ( self._process_spider_exception(response, ex, method_index + 1) @@ -421,7 +444,7 @@ class SpiderMiddlewareManager(MiddlewareManager): if self._use_start_requests: sync_start = iter(self._spider.start_requests()) sync_start = await self._process_chain( - "process_start_requests", sync_start, self._spider + "process_start_requests", sync_start, always_add_spider=True ) start: AsyncIterator[Any] = as_async_generator(sync_start) else: diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 8d2ade120..be41b52e4 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -8,7 +8,8 @@ from collections import defaultdict, deque from typing import TYPE_CHECKING, Any, TypeVar, cast from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.utils.defer import _process_chain, process_parallel +from scrapy.utils.defer import ensure_awaitable +from scrapy.utils.deprecate import argument_is_required from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.python import global_object_name @@ -31,7 +32,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) _T = TypeVar("_T") -_T2 = TypeVar("_T2") class MiddlewareManager(ABC): @@ -56,6 +56,7 @@ class MiddlewareManager(ABC): self.methods: dict[str, deque[Callable | tuple[Callable, Callable] | None]] = ( defaultdict(deque) ) + self._mw_methods_requiring_spider: set[Callable] = set() for mw in middlewares: self._add_middleware(mw) @@ -158,34 +159,53 @@ class MiddlewareManager(ABC): ) return cls(*middlewares, crawler=crawler) - def _add_middleware(self, mw: Any) -> None: - if hasattr(mw, "open_spider"): - self.methods["open_spider"].append(mw.open_spider) - if hasattr(mw, "close_spider"): - self.methods["close_spider"].appendleft(mw.close_spider) + def _add_middleware(self, mw: Any) -> None: # noqa: B027 + pass - def _process_parallel( - self, methodname: str, obj: _T, *args: Any - ) -> Deferred[list[_T2]]: - methods = cast( - "Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname] - ) - return process_parallel(methods, obj, *args) + def _check_mw_method_spider_arg(self, method: Callable) -> None: + if argument_is_required(method, "spider"): + warnings.warn( + f"{method.__qualname__}() requires a spider argument," + f" this is deprecated and the argument will not be passed in future Scrapy versions." + f" If you need to access the spider instance you can save the crawler instance" + f" passed to from_crawler() and use its spider attribute.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + self._mw_methods_requiring_spider.add(method) - async def _process_chain(self, methodname: str, obj: _T, *args: Any) -> _T: + async def _process_chain( + self, + methodname: str, + obj: _T, + *args: Any, + add_spider: bool = False, + always_add_spider: bool = False, + ) -> _T: methods = cast( "Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname] ) - return await _process_chain(methods, obj, *args) + for method in methods: + if always_add_spider or ( + add_spider and method in self._mw_methods_requiring_spider + ): + obj = await ensure_awaitable(method(obj, *(*args, self._spider))) + else: + obj = await ensure_awaitable(method(obj, *args)) + return obj - def open_spider(self, spider: Spider | None = None) -> Deferred[list[None]]: - if spider: - self._warn_spider_arg("open_spider") - self._set_compat_spider(spider) - return self._process_parallel("open_spider", self._spider) + def open_spider( + self, spider: Spider | None = None + ) -> Deferred[list[None]]: # pragma: no cover + raise NotImplementedError( + "MiddlewareManager.open_spider() is no longer implemented" + " and will be removed in a future Scrapy version." + ) - def close_spider(self, spider: Spider | None = None) -> Deferred[list[None]]: - if spider: - self._warn_spider_arg("close_spider") - self._set_compat_spider(spider) - return self._process_parallel("close_spider", self._spider) + def close_spider( + self, spider: Spider | None = None + ) -> Deferred[list[None]]: # pragma: no cover + raise NotImplementedError( + "MiddlewareManager.close_spider() is no longer implemented" + " and will be removed in a future Scrapy version." + ) diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 8fe40ab13..b7ec928e8 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -7,16 +7,20 @@ See documentation in docs/item-pipeline.rst from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Callable, cast + +from twisted.internet.defer import Deferred, DeferredList from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.middleware import MiddlewareManager from scrapy.utils.conf import build_component_list -from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.defer import deferred_from_coro, maybeDeferred_coro from scrapy.utils.python import global_object_name if TYPE_CHECKING: - from twisted.internet.defer import Deferred + from collections.abc import Iterable + + from twisted.python.failure import Failure from scrapy import Spider from scrapy.settings import Settings @@ -30,9 +34,15 @@ class ItemPipelineManager(MiddlewareManager): return build_component_list(settings.getwithbase("ITEM_PIPELINES")) def _add_middleware(self, pipe: Any) -> None: - super()._add_middleware(pipe) + if hasattr(pipe, "open_spider"): + self.methods["open_spider"].append(pipe.open_spider) + self._check_mw_method_spider_arg(pipe.open_spider) + if hasattr(pipe, "close_spider"): + self.methods["close_spider"].appendleft(pipe.close_spider) + self._check_mw_method_spider_arg(pipe.close_spider) if hasattr(pipe, "process_item"): self.methods["process_item"].append(pipe.process_item) + self._check_mw_method_spider_arg(pipe.process_item) def process_item(self, item: Any, spider: Spider | None = None) -> Deferred[Any]: if spider: @@ -45,4 +55,36 @@ class ItemPipelineManager(MiddlewareManager): return deferred_from_coro(self.process_item_async(item)) async def process_item_async(self, item: Any) -> Any: - return await self._process_chain("process_item", item, self._spider) + return await self._process_chain("process_item", item, add_spider=True) + + def _process_parallel(self, methodname: str) -> Deferred[list[None]]: + methods = cast("Iterable[Callable[..., None]]", self.methods[methodname]) + + def get_dfd(method: Callable[..., None]) -> Deferred[None]: + if method in self._mw_methods_requiring_spider: + return maybeDeferred_coro(method, self._spider) + return maybeDeferred_coro(method) + + dfds = [get_dfd(m) for m in methods] + d: Deferred[list[tuple[bool, None]]] = DeferredList( + dfds, fireOnOneErrback=True, consumeErrors=True + ) + d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r]) + + def eb(failure: Failure) -> Failure: + return failure.value.subFailure + + d2.addErrback(eb) + return d2 + + def open_spider(self, spider: Spider | None = None) -> Deferred[list[None]]: + if spider: + self._warn_spider_arg("open_spider") + self._set_compat_spider(spider) + return self._process_parallel("open_spider") + + def close_spider(self, spider: Spider | None = None) -> Deferred[list[None]]: + if spider: + self._warn_spider_arg("close_spider") + self._set_compat_spider(spider) + return self._process_parallel("close_spider") diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index a50150c59..fb65ff06a 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -165,12 +165,27 @@ class MediaPipeline(ABC): pipe._finish_init(crawler) return pipe - def open_spider(self, spider: Spider) -> None: - self.spiderinfo = self.SpiderInfo(spider) + def open_spider(self, spider: Spider | None = None) -> None: + if spider is not None: # pragma: no cover + warnings.warn( + "Passing a spider argument to MediaPipeline.open_spider()" + " is deprecated and the passed value is ignored.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + assert self.crawler.spider + self.spiderinfo = self.SpiderInfo(self.crawler.spider) def process_item( - self, item: Any, spider: Spider + self, item: Any, spider: Spider | None = None ) -> Deferred[list[FileInfoOrError]]: + if spider is not None: # pragma: no cover + warnings.warn( + "Passing a spider argument to MediaPipeline.process_item()" + " is deprecated and the passed value is ignored.", + ScrapyDeprecationWarning, + stacklevel=2, + ) info = self.spiderinfo requests = arg_to_iter(self.get_media_requests(item, info)) dlist = [self._process_request(r, info, item) for r in requests] diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index 196b84d0d..a047809c3 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -1,8 +1,10 @@ from __future__ import annotations +import warnings from typing import TYPE_CHECKING, Any from scrapy import Request, Spider +from scrapy.exceptions import ScrapyDeprecationWarning if TYPE_CHECKING: from collections.abc import AsyncIterator, Iterable @@ -53,15 +55,32 @@ class BaseSpiderMiddleware: yield o def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider + self, response: Response, result: Iterable[Any], spider: Spider | None = None ) -> Iterable[Any]: + if spider is not None: # pragma: no cover + warnings.warn( + "Passing a spider argument to BaseSpiderMiddleware.process_spider_output()" + " is deprecated and the passed value is ignored.", + ScrapyDeprecationWarning, + stacklevel=2, + ) for o in result: if (o := self._get_processed(o, response)) is not None: yield o async def process_spider_output_async( - self, response: Response, result: AsyncIterator[Any], spider: Spider + self, + response: Response, + result: AsyncIterator[Any], + spider: Spider | None = None, ) -> AsyncIterator[Any]: + if spider is not None: # pragma: no cover + warnings.warn( + "Passing a spider argument to BaseSpiderMiddleware.process_spider_output_async()" + " is deprecated and the passed value is ignored.", + ScrapyDeprecationWarning, + stacklevel=2, + ) async for o in result: if (o := self._get_processed(o, response)) is not None: yield o diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index fecdb0142..d23bca50c 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -7,8 +7,10 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging +import warnings from typing import TYPE_CHECKING, Any +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.spidermiddlewares.base import BaseSpiderMiddleware if TYPE_CHECKING: @@ -53,16 +55,33 @@ class DepthMiddleware(BaseSpiderMiddleware): return o def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider + self, response: Response, result: Iterable[Any], spider: Spider | None = None ) -> Iterable[Any]: + if spider is not None: # pragma: no cover + warnings.warn( + "Passing a spider argument to DepthMiddleware.process_spider_output()" + " is deprecated and the passed value is ignored.", + ScrapyDeprecationWarning, + stacklevel=2, + ) self._init_depth(response) - yield from super().process_spider_output(response, result, spider) + yield from super().process_spider_output(response, result) async def process_spider_output_async( - self, response: Response, result: AsyncIterator[Any], spider: Spider + self, + response: Response, + result: AsyncIterator[Any], + spider: Spider | None = None, ) -> AsyncIterator[Any]: + if spider is not None: # pragma: no cover + warnings.warn( + "Passing a spider argument to DepthMiddleware.process_spider_output_async()" + " is deprecated and the passed value is ignored.", + ScrapyDeprecationWarning, + stacklevel=2, + ) self._init_depth(response) - async for o in super().process_spider_output_async(response, result, spider): + async for o in super().process_spider_output_async(response, result): yield o def _init_depth(self, response: Response) -> None: diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 42619ec7f..b4c9c10bf 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -7,9 +7,10 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging +import warnings from typing import TYPE_CHECKING, Any -from scrapy.exceptions import IgnoreRequest +from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning if TYPE_CHECKING: from collections.abc import Iterable @@ -35,9 +36,7 @@ class HttpError(IgnoreRequest): class HttpErrorMiddleware: - @classmethod - def from_crawler(cls, crawler: Crawler) -> Self: - return cls(crawler.settings) + crawler: Crawler def __init__(self, settings: BaseSettings): self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL") @@ -45,7 +44,22 @@ class HttpErrorMiddleware: "HTTPERROR_ALLOWED_CODES" ) - def process_spider_input(self, response: Response, spider: Spider) -> None: + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + o = cls(crawler.settings) + o.crawler = crawler + return o + + def process_spider_input( + self, response: Response, spider: Spider | None = None + ) -> None: + if spider is not None: # pragma: no cover + warnings.warn( + "Passing a spider argument to HttpErrorMiddleware.process_spider_input()" + " is deprecated and the passed value is ignored.", + ScrapyDeprecationWarning, + stacklevel=2, + ) if 200 <= response.status < 300: # common case return meta = response.meta @@ -57,25 +71,34 @@ class HttpErrorMiddleware: return else: allowed_statuses = getattr( - spider, "handle_httpstatus_list", self.handle_httpstatus_list + self.crawler.spider, + "handle_httpstatus_list", + self.handle_httpstatus_list, ) if response.status in allowed_statuses: return raise HttpError(response, "Ignoring non-200 response") def process_spider_exception( - self, response: Response, exception: Exception, spider: Spider + self, response: Response, exception: Exception, spider: Spider | None = None ) -> Iterable[Any] | None: + if spider is not None: # pragma: no cover + warnings.warn( + "Passing a spider argument to HttpErrorMiddleware.process_spider_exception()" + " is deprecated and the passed value is ignored.", + ScrapyDeprecationWarning, + stacklevel=2, + ) if isinstance(exception, HttpError): - assert spider.crawler.stats - spider.crawler.stats.inc_value("httperror/response_ignored_count") - spider.crawler.stats.inc_value( + assert self.crawler.stats + self.crawler.stats.inc_value("httperror/response_ignored_count") + self.crawler.stats.inc_value( f"httperror/response_ignored_status_count/{response.status}" ) logger.info( "Ignoring response %(response)r: HTTP status code is not handled or not allowed", {"response": response}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) return [] return None diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index cbccbd244..7481e26ed 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -312,19 +312,6 @@ def process_chain( return d -async def _process_chain( - callables: Iterable[Callable[Concatenate[_T, _P], _T | Awaitable[_T]]], - input_: _T, - *a: _P.args, - **kw: _P.kwargs, -) -> _T: - """Chain the given (potentialy asynchronous) callables.""" - result = input_ - for callable_ in callables: - result = await ensure_awaitable(callable_(result, *a, **kw)) - return result - - def process_chain_both( callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]], @@ -355,10 +342,15 @@ def process_parallel( input: _T, # noqa: A002 *a: _P.args, **kw: _P.kwargs, -) -> Deferred[list[_T2]]: +) -> Deferred[list[_T2]]: # pragma: no cover """Return a Deferred with the output of all successful calls to the given callbacks """ + warnings.warn( + "process_parallel() is deprecated.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) dfds = [succeed(input).addCallback(x, *a, **kw) for x in callbacks] d: Deferred[list[tuple[bool, _T2]]] = DeferredList( dfds, fireOnOneErrback=True, consumeErrors=True diff --git a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py index c32aaf37d..e000a23db 100644 --- a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py +++ b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py @@ -16,7 +16,7 @@ class UppercasePipeline: def open_spider(self, spider): return deferred_from_coro(self._open_spider(spider)) - def process_item(self, item, spider): + def process_item(self, item): return {"url": item["url"].upper()} diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py index 028e3a08a..eb2e1105c 100644 --- a/tests/CrawlerProcess/asyncio_deferred_signal.py +++ b/tests/CrawlerProcess/asyncio_deferred_signal.py @@ -16,7 +16,7 @@ class UppercasePipeline: def open_spider(self, spider): return deferred_from_coro(self._open_spider(spider)) - def process_item(self, item, spider): + def process_item(self, item): return {"url": item["url"].upper()} diff --git a/tests/pipelines.py b/tests/pipelines.py index d9fc12676..4f00d6919 100644 --- a/tests/pipelines.py +++ b/tests/pipelines.py @@ -4,13 +4,13 @@ Some pipelines used for testing class ZeroDivisionErrorPipeline: - def open_spider(self, spider): + def open_spider(self): 1 / 0 - def process_item(self, item, spider): + def process_item(self, item): return item class ProcessWithZeroDivisionErrorPipeline: - def process_item(self, item, spider): + def process_item(self, item): 1 / 0 diff --git a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py index 3e29c70ed..577ecd686 100644 --- a/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py +++ b/tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py @@ -2,7 +2,7 @@ class TestSpiderPipeline: def open_spider(self, spider): pass - def process_item(self, item, spider): + def process_item(self, item): return item @@ -10,5 +10,5 @@ class TestSpiderExceptionPipeline: def open_spider(self, spider): raise RuntimeError("exception") - def process_item(self, item, spider): + def process_item(self, item): return item diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index fe7a807d4..7b119d7cf 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -153,7 +153,7 @@ import logging class MyPipeline: component_name = 'my_pipeline' - def process_item(self, item, spider): + def process_item(self, item): logging.info('It Works!') return item """, diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index f21e52e52..426be13a7 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -246,7 +246,7 @@ class SkipMessagesLogFormatter(LogFormatter): class DropSomeItemsPipeline: drop = True - def process_item(self, item, spider): + def process_item(self, item): if self.drop: self.drop = False raise DropItem("Ignoring item") diff --git a/tests/test_middleware.py b/tests/test_middleware.py index b6a07f872..6c3008b25 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -7,7 +7,6 @@ import pytest from scrapy import Spider from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.middleware import MiddlewareManager -from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler if TYPE_CHECKING: @@ -57,7 +56,10 @@ class MyMiddlewareManager(MiddlewareManager): return [M1, MOff, M3] def _add_middleware(self, mw): - super()._add_middleware(mw) + if hasattr(mw, "open_spider"): + self.methods["open_spider"].append(mw.open_spider) + if hasattr(mw, "close_spider"): + self.methods["close_spider"].appendleft(mw.close_spider) if hasattr(mw, "process"): self.methods["process"].append(mw.process) @@ -105,73 +107,3 @@ def test_no_crawler() -> None: mwman = MyMiddlewareManager(m1, m2, m3) assert mwman.middlewares == (m1, m2, m3) assert mwman.crawler is None - - -def test_deprecated_spider_arg_no_crawler_spider(crawler: Crawler) -> None: - """Crawler is provided, but doesn't have a spider. The instance passed to the method is - ignored and raises a warning.""" - mwman = MyMiddlewareManager(crawler=crawler) - with ( - pytest.warns( - ScrapyDeprecationWarning, - match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated", - ), - pytest.raises( - ValueError, - match="MyMiddlewareManager needs to access self.crawler.spider but it is None", - ), - ): - mwman.open_spider(DefaultSpider()) - - -def test_deprecated_spider_arg_with_crawler(crawler: Crawler) -> None: - """Crawler is provided and has a spider, works. The instance passed to the method is ignored, - even if mismatched, but raises a warning.""" - mwman = MyMiddlewareManager(crawler=crawler) - crawler.spider = crawler._create_spider("foo") - with pytest.warns( - ScrapyDeprecationWarning, - match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated", - ): - mwman.open_spider(DefaultSpider()) - with pytest.warns( - ScrapyDeprecationWarning, - match=r"Passing a spider argument to MyMiddlewareManager.close_spider\(\) is deprecated", - ): - mwman.close_spider(DefaultSpider()) - - -def test_deprecated_spider_arg_without_crawler() -> None: - """The first instance passed to the method is used, with a warning. Mismatched ones raise an error.""" - with pytest.warns( - ScrapyDeprecationWarning, - match="was called without the crawler argument", - ): - mwman = MyMiddlewareManager() - with pytest.warns( - ScrapyDeprecationWarning, - match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated", - ): - mwman.open_spider(DefaultSpider()) - with ( - pytest.warns( - ScrapyDeprecationWarning, - match=r"Passing a spider argument to MyMiddlewareManager.close_spider\(\) is deprecated", - ), - pytest.raises(RuntimeError, match="Different instances of Spider were passed"), - ): - mwman.close_spider(DefaultSpider()) - - -def test_no_spider_arg_without_crawler() -> None: - """If no crawler and no spider arg, raise an error.""" - with pytest.warns( - ScrapyDeprecationWarning, - match="was called without the crawler argument", - ): - mwman = MyMiddlewareManager() - with pytest.raises( - ValueError, - match="has no known Spider instance", - ): - mwman.open_spider() diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index a4cdaa5f8..db64589af 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -29,6 +29,7 @@ from scrapy.pipelines.files import ( GCSFilesStore, S3FilesStore, ) +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.ftp import MockFTPServer @@ -78,10 +79,11 @@ class TestFilesPipeline: def setup_method(self): self.tempdir = mkdtemp() settings_dict = {"FILES_STORE": self.tempdir} - crawler = get_crawler(spidercls=None, settings_dict=settings_dict) + crawler = get_crawler(DefaultSpider, settings_dict=settings_dict) + crawler.spider = crawler._create_spider() self.pipeline = FilesPipeline.from_crawler(crawler) self.pipeline.download_func = _mocked_download_func - self.pipeline.open_spider(None) + self.pipeline.open_spider() def teardown_method(self): rmtree(self.tempdir) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 42ae897e8..cc1d73349 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -13,10 +13,10 @@ from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException from scrapy.pipelines.media import MediaPipeline -from scrapy.spiders import Spider from scrapy.utils.asyncio import call_later from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler @@ -48,12 +48,11 @@ class TestBaseMediaPipeline: settings = None def setup_method(self): - spider_cls = Spider - self.spider = spider_cls("media.com") - crawler = get_crawler(spider_cls, self.settings) + crawler = get_crawler(DefaultSpider, self.settings) + crawler.spider = crawler._create_spider() self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe.download_func = _mocked_download_func - self.pipe.open_spider(self.spider) + self.pipe.open_spider() self.info = self.pipe.spiderinfo self.fingerprint = crawler.request_fingerprinter.fingerprint @@ -162,7 +161,7 @@ class TestBaseMediaPipeline: @inlineCallbacks def test_default_process_item(self): item = {"name": "name"} - new_item = yield self.pipe.process_item(item, self.spider) + new_item = yield self.pipe.process_item(item) assert new_item is item @@ -216,7 +215,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): errback=self._errback, ) item = {"requests": req} - new_item = yield self.pipe.process_item(item, self.spider) + new_item = yield self.pipe.process_item(item) assert new_item["results"] == [(True, {})] assert self.pipe._mockcalled == [ "get_media_requests", @@ -236,7 +235,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): errback=self._errback, ) item = {"requests": req} - new_item = yield self.pipe.process_item(item, self.spider) + new_item = yield self.pipe.process_item(item) assert len(new_item["results"]) == 1 assert new_item["results"][0][0] is False assert isinstance(new_item["results"][0][1], Failure) @@ -258,7 +257,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): fail = Failure(exc) req2 = Request("http://url2", meta={"response": fail}) item = {"requests": [req1, req2]} - new_item = yield self.pipe.process_item(item, self.spider) + new_item = yield self.pipe.process_item(item) assert len(new_item["results"]) == 2 assert new_item["results"][0] == (True, {}) assert new_item["results"][1][0] is False @@ -281,7 +280,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): # returns single Request (without callback) req = Request("http://url") item = {"requests": req} # pass a single item - new_item = yield self.pipe.process_item(item, self.spider) + new_item = yield self.pipe.process_item(item) assert new_item is item assert self.fingerprint(req) in self.info.downloaded @@ -289,7 +288,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): req1 = Request("http://url1") req2 = Request("http://url2") item = {"requests": iter([req1, req2])} - new_item = yield self.pipe.process_item(item, self.spider) + new_item = yield self.pipe.process_item(item) assert new_item is item assert self.fingerprint(req1) in self.info.downloaded assert self.fingerprint(req2) in self.info.downloaded @@ -299,7 +298,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): rsp1 = Response("http://url1") req1 = Request("http://url1", meta={"response": rsp1}) item = {"requests": req1} - new_item = yield self.pipe.process_item(item, self.spider) + new_item = yield self.pipe.process_item(item) assert new_item is item assert new_item["results"] == [(True, {})] @@ -308,7 +307,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): req1.url, meta={"response": Response("http://donot.download.me")} ) item = {"requests": req2} - new_item = yield self.pipe.process_item(item, self.spider) + new_item = yield self.pipe.process_item(item) assert new_item is item assert self.fingerprint(req1) == self.fingerprint(req2) assert new_item["results"] == [(True, {})] @@ -321,7 +320,7 @@ class TestMediaPipeline(TestBaseMediaPipeline): req1.url, meta={"response": Response("http://donot.download.me")} ) item = {"requests": [req1, req2]} - new_item = yield self.pipe.process_item(item, self.spider) + new_item = yield self.pipe.process_item(item) assert new_item is item assert new_item["results"] == [(True, {}), (True, {})] @@ -348,14 +347,14 @@ class TestMediaPipeline(TestBaseMediaPipeline): req1 = Request("http://url", meta={"response": rsp1_func}) req2 = Request(req1.url, meta={"response": rsp2_func}) item = {"requests": [req1, req2]} - new_item = yield self.pipe.process_item(item, self.spider) + new_item = yield self.pipe.process_item(item) assert new_item["results"] == [(True, {}), (True, {})] @inlineCallbacks def test_use_media_to_download_result(self): req = Request("http://url", meta={"result": "ITSME"}) item = {"requests": req} - new_item = yield self.pipe.process_item(item, self.spider) + new_item = yield self.pipe.process_item(item) assert new_item["results"] == [(True, "ITSME")] assert self.pipe._mockcalled == [ "get_media_requests", @@ -555,12 +554,11 @@ class TestMediaFailedFailure: settings = None def setup_method(self): - spider_cls = Spider - self.spider = spider_cls("media.com") - crawler = get_crawler(spider_cls, self.settings) + crawler = get_crawler(DefaultSpider, self.settings) + crawler.spider = crawler._create_spider() self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe.download_func = _mocked_download_func - self.pipe.open_spider(self.spider) + self.pipe.open_spider() self.info = self.pipe.spiderinfo self.fingerprint = crawler.request_fingerprinter.fingerprint @@ -587,7 +585,7 @@ class TestMediaFailedFailure: with pytest.warns( ScrapyDeprecationWarning, match="media_failed returned a Failure instance" ): - new_item = yield self.pipe.process_item(item, self.spider) + new_item = yield self.pipe.process_item(item) assert len(new_item["results"]) == 1 assert new_item["results"][0][0] is False assert isinstance(new_item["results"][0][1], Failure) diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 725ceccee..4b8007ead 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -4,6 +4,7 @@ import pytest from twisted.internet.defer import Deferred, inlineCallbacks, succeed from scrapy import Request, Spider, signals +from scrapy.crawler import Crawler from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.pipelines import ItemPipelineManager from scrapy.utils.asyncio import call_later @@ -19,17 +20,25 @@ from tests.mockserver.http import MockServer class SimplePipeline: - def process_item(self, item, spider): + def process_item(self, item): item["pipeline_passed"] = True return item +class DeprecatedSpiderArgPipeline: + def open_spider(self, spider): + pass + + def close_spider(self, spider): + pass + + class DeferredPipeline: def cb(self, item): item["pipeline_passed"] = True return item - def process_item(self, item, spider): + def process_item(self, item): d = Deferred() d.addCallback(self.cb) d.callback(item) @@ -37,7 +46,7 @@ class DeferredPipeline: class AsyncDefPipeline: - async def process_item(self, item, spider): + async def process_item(self, item): d = Deferred() call_later(0, d.callback, None) await maybe_deferred_to_future(d) @@ -46,7 +55,7 @@ class AsyncDefPipeline: class AsyncDefAsyncioPipeline: - async def process_item(self, item, spider): + async def process_item(self, item): d = Deferred() loop = asyncio.get_event_loop() loop.call_later(0, d.callback, None) @@ -57,7 +66,7 @@ class AsyncDefAsyncioPipeline: class AsyncDefNotAsyncioPipeline: - async def process_item(self, item, spider): + async def process_item(self, item): d1 = Deferred() from twisted.internet import reactor @@ -184,6 +193,9 @@ class TestCustomPipelineManager: def open_spider(self, spider): # pylint: disable=signature-differs return super().open_spider(spider) + def close_spider(self, spider): # pylint: disable=signature-differs + return super().close_spider(spider) + def process_item(self, item, spider): # pylint: disable=signature-differs with pytest.warns( ScrapyDeprecationWarning, @@ -212,6 +224,10 @@ class TestCustomPipelineManager: ScrapyDeprecationWarning, match=r"The open_spider\(\) method of .+\.CustomPipelineManager requires a spider argument", ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"The close_spider\(\) method of .+\.CustomPipelineManager requires a spider argument", + ), pytest.warns( ScrapyDeprecationWarning, match=r"The process_item\(\) method of .+\.CustomPipelineManager requires a spider argument", @@ -220,6 +236,10 @@ class TestCustomPipelineManager: ScrapyDeprecationWarning, match=r"Passing a spider argument to CustomPipelineManager.open_spider\(\) is deprecated", ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a spider argument to CustomPipelineManager.close_spider\(\) is deprecated", + ), pytest.warns( ScrapyDeprecationWarning, match=r"CustomPipelineManager overrides process_item\(\) but doesn't override process_item_async\(\)", @@ -254,7 +274,7 @@ class TestCustomPipelineManager: def process_item(self, item, spider): for pipeline in self.pipelines: - item = pipeline.process_item(item, spider) + item = pipeline.process_item(item) return succeed(item) items = [] @@ -294,3 +314,128 @@ class TestCustomPipelineManager: await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver)) assert len(items) == 1 + + +class TestMiddlewareManagerSpider: + """Tests for the deprecated spider arg handling in MiddlewareManager. + + Here because MiddlewareManager doesn't have methods that could take a spider arg.""" + + @pytest.fixture + def crawler(self) -> Crawler: + return get_crawler(Spider) + + def test_deprecated_spider_arg_no_crawler_spider(self, crawler: Crawler) -> None: + """Crawler is provided, but doesn't have a spider. The instance passed to the method is + ignored and raises a warning.""" + mwman = ItemPipelineManager(crawler=crawler) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument", + ), + ): + mwman._add_middleware(DeprecatedSpiderArgPipeline()) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated", + ), + pytest.raises( + ValueError, + match="ItemPipelineManager needs to access self.crawler.spider but it is None", + ), + ): + mwman.open_spider(DefaultSpider()) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated", + ), + pytest.raises( + ValueError, + match="ItemPipelineManager needs to access self.crawler.spider but it is None", + ), + ): + mwman.close_spider(DefaultSpider()) + + def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None: + """Crawler is provided and has a spider, works. The instance passed to the method is ignored, + even if mismatched, but raises a warning.""" + mwman = ItemPipelineManager(crawler=crawler) + crawler.spider = crawler._create_spider("foo") + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated", + ): + mwman.open_spider(DefaultSpider()) + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated", + ): + mwman.close_spider(DefaultSpider()) + + def test_deprecated_spider_arg_without_crawler(self) -> None: + """The first instance passed to the method is used, with a warning. Mismatched ones raise an error.""" + with pytest.warns( + ScrapyDeprecationWarning, + match="was called without the crawler argument", + ): + mwman = ItemPipelineManager() + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument", + ), + ): + mwman._add_middleware(DeprecatedSpiderArgPipeline()) + with pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated", + ): + mwman.open_spider(DefaultSpider()) + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated", + ), + pytest.raises( + RuntimeError, match="Different instances of Spider were passed" + ), + ): + mwman.close_spider(DefaultSpider()) + mwman.close_spider() + + def test_no_spider_arg_without_crawler(self) -> None: + """If no crawler and no spider arg, raise an error.""" + with pytest.warns( + ScrapyDeprecationWarning, + match="was called without the crawler argument", + ): + mwman = ItemPipelineManager() + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument", + ), + ): + mwman._add_middleware(DeprecatedSpiderArgPipeline()) + with ( + pytest.raises( + ValueError, + match="has no known Spider instance", + ), + ): + mwman.open_spider() diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 041fc9588..0f31af8ad 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -33,12 +33,12 @@ class InjectArgumentsSpiderMiddleware: request.cb_kwargs["from_process_start"] = True yield request - def process_spider_input(self, response, spider): + def process_spider_input(self, response): request = response.request if request.callback.__name__ == "parse_spider_mw": request.cb_kwargs["from_process_spider_input"] = True - def process_spider_output(self, response, result, spider): + def process_spider_output(self, response, result): for element in result: if ( isinstance(element, Request) diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 76609ac52..c1f358bd2 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -451,7 +451,7 @@ class TestSettings: def test_passing_objects_as_values(self): class TestPipeline: - def process_item(self, i, s): + def process_item(self, i): return i settings = Settings( @@ -471,7 +471,7 @@ class TestSettings: assert priority == 800 assert mypipeline == TestPipeline assert isinstance(mypipeline(), TestPipeline) - assert mypipeline().process_item("item", None) == "item" + assert mypipeline().process_item("item") == "item" myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp") assert myhandler == FileDownloadHandler diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 2ef810852..8fd33eb13 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -10,7 +10,7 @@ from testfixtures import LogCapture from twisted.internet import defer from scrapy.core.spidermw import SpiderMiddlewareManager -from scrapy.exceptions import _InvalidOutput +from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen @@ -54,7 +54,7 @@ class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): @deferred_f_from_coro_f async def test_invalid_process_spider_input(self): class InvalidProcessSpiderInputMiddleware: - def process_spider_input(self, response, spider): + def process_spider_input(self, response): return 1 self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware()) @@ -68,7 +68,7 @@ class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): @deferred_f_from_coro_f async def test_invalid_process_spider_output(self): class InvalidProcessSpiderOutputMiddleware: - def process_spider_output(self, response, result, spider): + def process_spider_output(self, response, result): return 1 self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) @@ -82,11 +82,11 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): @deferred_f_from_coro_f async def test_invalid_process_spider_exception(self): class InvalidProcessSpiderOutputExceptionMiddleware: - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): return 1 class RaiseExceptionProcessSpiderOutputMiddleware: - def process_spider_output(self, response, result, spider): + def process_spider_output(self, response, result): raise RuntimeError self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) @@ -101,11 +101,11 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): @deferred_f_from_coro_f async def test_process_spider_exception_return_none(self): class ProcessSpiderExceptionReturnNoneMiddleware: - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): return None class RaiseExceptionProcessSpiderOutputMiddleware: - def process_spider_output(self, response, result, spider): + def process_spider_output(self, response, result): 1 / 0 self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) @@ -191,34 +191,34 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): class ProcessSpiderOutputSimpleMiddleware: - def process_spider_output(self, response, result, spider): + def process_spider_output(self, response, result): yield from result class ProcessSpiderOutputAsyncGenMiddleware: - async def process_spider_output(self, response, result, spider): + async def process_spider_output(self, response, result): async for r in result: yield r class ProcessSpiderOutputUniversalMiddleware: - def process_spider_output(self, response, result, spider): + def process_spider_output(self, response, result): yield from result - async def process_spider_output_async(self, response, result, spider): + async def process_spider_output_async(self, response, result): async for r in result: yield r class ProcessSpiderExceptionSimpleIterableMiddleware: - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): yield {"foo": 1} yield {"foo": 2} yield {"foo": 3} class ProcessSpiderExceptionAsyncIteratorMiddleware: - async def process_spider_exception(self, response, exception, spider): + async def process_spider_exception(self, response, exception): yield {"foo": 1} d = defer.Deferred() call_later(0, d.callback, None) @@ -315,12 +315,12 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): class ProcessSpiderOutputNonIterableMiddleware: - def process_spider_output(self, response, result, spider): + def process_spider_output(self, response, result): return class ProcessSpiderOutputCoroutineMiddleware: - async def process_spider_output(self, response, result, spider): + async def process_spider_output(self, response, result): return result @@ -384,23 +384,23 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware): class UniversalMiddlewareNoSync: - async def process_spider_output_async(self, response, result, spider): + async def process_spider_output_async(self, response, result): yield class UniversalMiddlewareBothSync: - def process_spider_output(self, response, result, spider): + def process_spider_output(self, response, result): yield - def process_spider_output_async(self, response, result, spider): + def process_spider_output_async(self, response, result): yield class UniversalMiddlewareBothAsync: - async def process_spider_output(self, response, result, spider): + async def process_spider_output(self, response, result): yield - async def process_spider_output_async(self, response, result, spider): + async def process_spider_output_async(self, response, result): yield @@ -600,3 +600,34 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): async def test_exc_async_simple(self): """Async exc mw -> simple output mw; cannot work as downgrading is not supported""" await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN) + + +class TestDeprecatedSpiderArg(TestSpiderMiddleware): + @deferred_f_from_coro_f + async def test_deprecated_spider_arg(self): + class DeprecatedSpiderArgMiddleware: + def process_spider_input(self, response, spider): + return None + + def process_spider_output(self, response, result, spider): + 1 / 0 + + def process_spider_exception(self, response, exception, spider): + return [] + + with ( + pytest.warns( + ScrapyDeprecationWarning, + match=r"process_spider_input\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"process_spider_output\(\) requires a spider argument", + ), + pytest.warns( + ScrapyDeprecationWarning, + match=r"process_spider_exception\(\) requires a spider argument", + ), + ): + self.mwman._add_middleware(DeprecatedSpiderArgMiddleware()) + await self._scrape_response() diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index 03446db7e..00e547305 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -21,12 +21,6 @@ def crawler() -> Crawler: return get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True}) -@pytest.fixture -def spider(crawler: Crawler) -> Spider: - crawler.spider = crawler._create_spider("scrapytest.org") - return crawler.spider - - @pytest.fixture def stats(crawler: Crawler) -> Generator[StatsCollector]: assert crawler.stats is not None @@ -42,15 +36,13 @@ def mw(crawler: Crawler) -> DepthMiddleware: return DepthMiddleware.from_crawler(crawler) -def test_process_spider_output( - mw: DepthMiddleware, stats: StatsCollector, spider: Spider -) -> None: +def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> None: req = Request("http://scrapytest.org") resp = Response("http://scrapytest.org") resp.request = req result = [Request("http://scrapytest.org")] - out = list(mw.process_spider_output(resp, result, spider)) + out = list(mw.process_spider_output(resp, result)) assert out == result rdc = stats.get_value("request_depth_count/1") @@ -58,7 +50,7 @@ def test_process_spider_output( req.meta["depth"] = 1 - out2 = list(mw.process_spider_output(resp, result, spider)) + out2 = list(mw.process_spider_output(resp, result)) assert not out2 rdm = stats.get_value("request_depth_max") diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index b41ae986e..9899a0cd7 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -7,9 +7,8 @@ from testfixtures import LogCapture from twisted.internet.defer import inlineCallbacks from scrapy.http import Request, Response -from scrapy.settings import Settings from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware -from scrapy.spiders import Spider +from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import MockServerSpider @@ -57,12 +56,6 @@ def _response(request: Request, status_code: int) -> Response: return Response(request.url, status=status_code, request=request) -@pytest.fixture -def spider() -> Spider: - crawler = get_crawler(Spider) - return Spider.from_crawler(crawler, name="foo") - - @pytest.fixture def res200() -> Response: return _response(req, 200) @@ -81,36 +74,35 @@ def res404() -> Response: class TestHttpErrorMiddleware: @pytest.fixture def mw(self) -> HttpErrorMiddleware: - return HttpErrorMiddleware(Settings({})) + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + return HttpErrorMiddleware.from_crawler(crawler) def test_process_spider_input( - self, - mw: HttpErrorMiddleware, - spider: Spider, - res200: Response, - res404: Response, + self, mw: HttpErrorMiddleware, res200: Response, res404: Response ) -> None: - mw.process_spider_input(res200, spider) + mw.process_spider_input(res200) with pytest.raises(HttpError): - mw.process_spider_input(res404, spider) + mw.process_spider_input(res404) def test_process_spider_exception( - self, mw: HttpErrorMiddleware, spider: Spider, res404: Response + self, mw: HttpErrorMiddleware, res404: Response ) -> None: - assert mw.process_spider_exception(res404, HttpError(res404), spider) == [] - assert mw.process_spider_exception(res404, Exception(), spider) is None + assert mw.process_spider_exception(res404, HttpError(res404)) == [] + assert mw.process_spider_exception(res404, Exception()) is None def test_handle_httpstatus_list( - self, mw: HttpErrorMiddleware, spider: Spider, res404: Response + self, mw: HttpErrorMiddleware, res404: Response ) -> None: request = Request( "http://scrapytest.org", meta={"handle_httpstatus_list": [404]} ) res = _response(request, 404) - mw.process_spider_input(res, spider) + mw.process_spider_input(res) - spider.handle_httpstatus_list = [404] # type: ignore[attr-defined] - mw.process_spider_input(res404, spider) + assert mw.crawler.spider + mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined] + mw.process_spider_input(res404) class TestHttpErrorMiddlewareSettings: @@ -118,76 +110,71 @@ class TestHttpErrorMiddlewareSettings: @pytest.fixture def mw(self) -> HttpErrorMiddleware: - return HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)})) + crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOWED_CODES": (402,)}) + crawler.spider = crawler._create_spider() + return HttpErrorMiddleware.from_crawler(crawler) def test_process_spider_input( self, mw: HttpErrorMiddleware, - spider: Spider, res200: Response, res402: Response, res404: Response, ) -> None: - mw.process_spider_input(res200, spider) + mw.process_spider_input(res200) with pytest.raises(HttpError): - mw.process_spider_input(res404, spider) - mw.process_spider_input(res402, spider) + mw.process_spider_input(res404) + mw.process_spider_input(res402) - def test_meta_overrides_settings( - self, mw: HttpErrorMiddleware, spider: Spider - ) -> None: + def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None: request = Request( "http://scrapytest.org", meta={"handle_httpstatus_list": [404]} ) res404 = _response(request, 404) res402 = _response(request, 402) - mw.process_spider_input(res404, spider) + mw.process_spider_input(res404) with pytest.raises(HttpError): - mw.process_spider_input(res402, spider) + mw.process_spider_input(res402) def test_spider_override_settings( - self, - mw: HttpErrorMiddleware, - spider: Spider, - res402: Response, - res404: Response, + self, mw: HttpErrorMiddleware, res402: Response, res404: Response ) -> None: - spider.handle_httpstatus_list = [404] # type: ignore[attr-defined] - mw.process_spider_input(res404, spider) + assert mw.crawler.spider + mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined] + mw.process_spider_input(res404) with pytest.raises(HttpError): - mw.process_spider_input(res402, spider) + mw.process_spider_input(res402) class TestHttpErrorMiddlewareHandleAll: @pytest.fixture def mw(self) -> HttpErrorMiddleware: - return HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True})) + crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOW_ALL": True}) + crawler.spider = crawler._create_spider() + return HttpErrorMiddleware.from_crawler(crawler) def test_process_spider_input( self, mw: HttpErrorMiddleware, - spider: Spider, res200: Response, res404: Response, ) -> None: - mw.process_spider_input(res200, spider) - mw.process_spider_input(res404, spider) + mw.process_spider_input(res200) + mw.process_spider_input(res404) - def test_meta_overrides_settings( - self, mw: HttpErrorMiddleware, spider: Spider - ) -> None: + def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None: request = Request( "http://scrapytest.org", meta={"handle_httpstatus_list": [404]} ) res404 = _response(request, 404) res402 = _response(request, 402) - mw.process_spider_input(res404, spider) + mw.process_spider_input(res404) with pytest.raises(HttpError): - mw.process_spider_input(res402, spider) + mw.process_spider_input(res402) - def test_httperror_allow_all_false(self, spider: Spider) -> None: + def test_httperror_allow_all_false(self) -> None: crawler = get_crawler(_HttpErrorSpider) mw = HttpErrorMiddleware.from_crawler(crawler) request_httpstatus_false = Request( @@ -200,8 +187,8 @@ class TestHttpErrorMiddlewareHandleAll: res402 = _response(request_httpstatus_true, 402) with pytest.raises(HttpError): - mw.process_spider_input(res404, spider) - mw.process_spider_input(res402, spider) + mw.process_spider_input(res404) + mw.process_spider_input(res402) class TestHttpErrorMiddlewareIntegrational: @@ -218,7 +205,7 @@ class TestHttpErrorMiddlewareIntegrational: def test_middleware_works(self): crawler = get_crawler(_HttpErrorSpider) yield crawler.crawl(mockserver=self.mockserver) - assert not crawler.spider.skipped, crawler.spider.skipped + assert not crawler.spider.skipped assert crawler.spider.parsed == {"200"} assert crawler.spider.failed == {"404", "402", "500"} diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 3f2b2057b..1808e087c 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -6,18 +6,27 @@ from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer -class LogExceptionMiddleware: - def process_spider_exception(self, response, exception, spider): - spider.logger.info( +class _BaseSpiderMiddleware: + def __init__(self, crawler): + self.crawler = crawler + + @classmethod + def from_crawler(cls, crawler): + return cls(crawler) + + +class LogExceptionMiddleware(_BaseSpiderMiddleware): + def process_spider_exception(self, response, exception): + self.crawler.spider.logger.info( "Middleware: %s exception caught", exception.__class__.__name__ ) # ================================================================================ # (0) recover from an exception on a spider callback -class RecoveryMiddleware: - def process_spider_exception(self, response, exception, spider): - spider.logger.info( +class RecoveryMiddleware(_BaseSpiderMiddleware): + def process_spider_exception(self, response, exception): + self.crawler.spider.logger.info( "Middleware: %s exception caught", exception.__class__.__name__ ) return [ @@ -55,9 +64,9 @@ class RecoveryAsyncGenSpider(RecoverySpider): # ================================================================================ # (1) exceptions from a spider middleware's process_spider_input method -class FailProcessSpiderInputMiddleware: - def process_spider_input(self, response, spider): - spider.logger.info("Middleware: will raise IndexError") +class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware): + def process_spider_input(self, response): + self.crawler.spider.logger.info("Middleware: will raise IndexError") raise IndexError @@ -159,27 +168,31 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS # ================================================================================ # (4) exceptions from a middleware process_spider_output method (generator) -class _GeneratorDoNothingMiddleware: - def process_spider_output(self, response, result, spider): +class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware): + def process_spider_output(self, response, result): for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): method = f"{self.__class__.__name__}.process_spider_exception" - spider.logger.info("%s: %s caught", method, exception.__class__.__name__) + self.crawler.spider.logger.info( + "%s: %s caught", method, exception.__class__.__name__ + ) -class GeneratorFailMiddleware: - def process_spider_output(self, response, result, spider): +class GeneratorFailMiddleware(_BaseSpiderMiddleware): + def process_spider_output(self, response, result): for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r raise LookupError - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): method = f"{self.__class__.__name__}.process_spider_exception" - spider.logger.info("%s: %s caught", method, exception.__class__.__name__) + self.crawler.spider.logger.info( + "%s: %s caught", method, exception.__class__.__name__ + ) yield {"processed": [method]} @@ -187,15 +200,17 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware): pass -class GeneratorRecoverMiddleware: - def process_spider_output(self, response, result, spider): +class GeneratorRecoverMiddleware(_BaseSpiderMiddleware): + def process_spider_output(self, response, result): for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): method = f"{self.__class__.__name__}.process_spider_exception" - spider.logger.info("%s: %s caught", method, exception.__class__.__name__) + self.crawler.spider.logger.info( + "%s: %s caught", method, exception.__class__.__name__ + ) yield {"processed": [method]} @@ -226,30 +241,34 @@ class GeneratorOutputChainSpider(Spider): # (5) exceptions from a middleware process_spider_output method (not generator) -class _NotGeneratorDoNothingMiddleware: - def process_spider_output(self, response, result, spider): +class _NotGeneratorDoNothingMiddleware(_BaseSpiderMiddleware): + def process_spider_output(self, response, result): out = [] for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") out.append(r) return out - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): method = f"{self.__class__.__name__}.process_spider_exception" - spider.logger.info("%s: %s caught", method, exception.__class__.__name__) + self.crawler.spider.logger.info( + "%s: %s caught", method, exception.__class__.__name__ + ) -class NotGeneratorFailMiddleware: - def process_spider_output(self, response, result, spider): +class NotGeneratorFailMiddleware(_BaseSpiderMiddleware): + def process_spider_output(self, response, result): out = [] for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") out.append(r) raise ReferenceError - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): method = f"{self.__class__.__name__}.process_spider_exception" - spider.logger.info("%s: %s caught", method, exception.__class__.__name__) + self.crawler.spider.logger.info( + "%s: %s caught", method, exception.__class__.__name__ + ) return [{"processed": [method]}] @@ -257,17 +276,19 @@ class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddlewa pass -class NotGeneratorRecoverMiddleware: - def process_spider_output(self, response, result, spider): +class NotGeneratorRecoverMiddleware(_BaseSpiderMiddleware): + def process_spider_output(self, response, result): out = [] for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") out.append(r) return out - def process_spider_exception(self, response, exception, spider): + def process_spider_exception(self, response, exception): method = f"{self.__class__.__name__}.process_spider_exception" - spider.logger.info("%s: %s caught", method, exception.__class__.__name__) + self.crawler.spider.logger.info( + "%s: %s caught", method, exception.__class__.__name__ + ) return [{"processed": [method]}] diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py index 895df1804..658351999 100644 --- a/tests/test_spidermiddleware_referer.py +++ b/tests/test_spidermiddleware_referer.py @@ -61,11 +61,11 @@ class TestRefererMiddleware: def get_response(self, origin: str) -> Response: return Response(origin, headers=self.resp_headers) - def test(self, mw: RefererMiddleware, spider: Spider) -> None: + def test(self, mw: RefererMiddleware) -> None: for origin, target, referrer in self.scenarii: response = self.get_response(origin) request = self.get_request(target) - out = list(mw.process_spider_output(response, [request], spider)) + out = list(mw.process_spider_output(response, [request])) assert out[0].headers.get("Referer") == referrer @@ -1036,7 +1036,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware): response = self.get_response(parent) request = self.get_request(target) - out = list(referrermw.process_spider_output(response, [request], spider)) + out = list(referrermw.process_spider_output(response, [request])) assert out[0].headers.get("Referer") == init_referrer for status, url in redirections: diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py index ca65529a3..750ae3b07 100644 --- a/tests/test_spidermiddleware_urllength.py +++ b/tests/test_spidermiddleware_urllength.py @@ -27,11 +27,6 @@ def crawler() -> Crawler: return get_crawler(Spider, {"URLLENGTH_LIMIT": maxlength}) -@pytest.fixture -def spider(crawler: Crawler) -> Spider: - return crawler._create_spider("foo") - - @pytest.fixture def stats(crawler: Crawler) -> StatsCollector: assert crawler.stats is not None @@ -43,22 +38,19 @@ def mw(crawler: Crawler) -> UrlLengthMiddleware: return UrlLengthMiddleware.from_crawler(crawler) -def process_spider_output(mw: UrlLengthMiddleware, spider: Spider) -> list[Request]: - return list(mw.process_spider_output(response, reqs, spider)) +def process_spider_output(mw: UrlLengthMiddleware) -> list[Request]: + return list(mw.process_spider_output(response, reqs)) -def test_middleware_works(mw: UrlLengthMiddleware, spider: Spider) -> None: - assert process_spider_output(mw, spider) == [short_url_req] +def test_middleware_works(mw: UrlLengthMiddleware) -> None: + assert process_spider_output(mw) == [short_url_req] def test_logging( - stats: StatsCollector, - mw: UrlLengthMiddleware, - spider: Spider, - caplog: pytest.LogCaptureFixture, + stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture ) -> None: with caplog.at_level(INFO): - process_spider_output(mw, spider) + process_spider_output(mw) ric = stats.get_value("urllength/request_ignored_count") assert ric == 1 assert f"Ignoring link (url length > {maxlength})" in caplog.text diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 7570fb96c..df7f1a20e 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -10,7 +10,6 @@ from twisted.internet.defer import Deferred, inlineCallbacks, succeed from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import ( - _process_chain, aiter_errback, deferred_f_from_coro_f, deferred_from_coro, @@ -19,7 +18,6 @@ from scrapy.utils.defer import ( maybe_deferred_to_future, mustbe_deferred, parallel_async, - process_parallel, ) if TYPE_CHECKING: @@ -85,26 +83,6 @@ def eb1(failure, arg1, arg2): return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})" -class TestDeferUtils: - @deferred_f_from_coro_f - async def test_process_chain(self): - x = await _process_chain([cb1, cb2, cb3], "res", "v1", "v2") - assert x == "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)" - - with pytest.raises(TypeError): - await _process_chain([cb1, cb_fail, cb3], "res", "v1", "v2") - - @inlineCallbacks - def test_process_parallel(self): - x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2") - assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"] - - @inlineCallbacks - def test_process_parallel_failure(self): - with pytest.raises(TypeError): - yield process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2") - - class TestIterErrback: def test_iter_errback_good(self): def itergood() -> Generator[int, None, None]: