RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Deprecate spider args of middlewares and pipelines. (#7006) 

* Deprecate spider args of spider middleware methods.

* Deprecate the spider arg of pipeline process_item().

* Fix a typing issue.

* Deprecate the spider arg of pipeline {open,close}_spider().

* Cleanup.

* Update docs.

* Add pragma: no cover to some deprecated code.

* More tests.

* More tests.

* More tests.

* Simplify _process_parallel().
This commit is contained in:
Andrey Rakhmatullin 2025-08-21 13:23:10 +05:00 committed by GitHub
parent b68330811b
commit ed63fa94d6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
34 changed files with 617 additions and 383 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/faq.rst
View File

@ -349,7 +349,7 @@ method for this purpose. For example:
class MultiplyItemsMiddleware: class MultiplyItemsMiddleware:
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
for item_or_request in result: for item_or_request in result:
if isinstance(item_or_request, Request): if isinstance(item_or_request, Request):
continue continue

14
docs/news.rst
View File

@ -44,6 +44,20 @@ Backward-incompatible changes
- :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer` - :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer`
- :class:`scrapy.middleware.MiddlewareManager` no longer includes code for
handling ``open_spider()`` and ``close_spider()`` component methods. As
this code was only used for pipelines it was moved into
:class:`scrapy.pipelines.ItemPipelineManager`. This change should only
affect custom subclasses of :class:`~scrapy.middleware.MiddlewareManager`.
The following code was moved:
- ``scrapy.middleware.MiddlewareManager.open_spider()``
- ``scrapy.middleware.MiddlewareManager.close_spider()``
- Code in ``scrapy.middleware.MiddlewareManager._add_middleware()`` that
processes ``open_spider()`` and ``close_spider()`` component methods.
- :meth:`scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware.process_request` - :meth:`scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware.process_request`
now returns a coroutine, previously it returned a now returns a coroutine, previously it returned a
:class:`~twisted.internet.defer.Deferred` object or ``None``. The :class:`~twisted.internet.defer.Deferred` object or ``None``. The

8
docs/topics/coroutines.rst
View File

@ -191,7 +191,7 @@ shorter and cleaner:
adapter["field"] = data adapter["field"] = data
return item return item
def process_item(self, item, spider): def process_item(self, item):
adapter = ItemAdapter(item) adapter = ItemAdapter(item)
dfd = db.get_some_data(adapter["id"]) dfd = db.get_some_data(adapter["id"])
dfd.addCallback(self._update_item, item) dfd.addCallback(self._update_item, item)
@ -205,7 +205,7 @@ becomes:
class DbPipeline: class DbPipeline:
async def process_item(self, item, spider): async def process_item(self, item):
adapter = ItemAdapter(item) adapter = ItemAdapter(item)
adapter["field"] = await db.get_some_data(adapter["id"]) adapter["field"] = await db.get_some_data(adapter["id"])
return item return item
@ -421,12 +421,12 @@ For example:
.. code-block:: python .. code-block:: python
class UniversalSpiderMiddleware: class UniversalSpiderMiddleware:
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
for r in result: for r in result:
# ... do something with r # ... do something with r
yield r yield r
async def process_spider_output_async(self, response, result, spider): async def process_spider_output_async(self, response, result):
async for r in result: async for r in result:
# ... do something with r # ... do something with r
yield r yield r

2
docs/topics/exporters.rst
View File

@ -67,7 +67,7 @@ value of one of their fields:
self.year_to_exporter[year] = (exporter, xml_file) self.year_to_exporter[year] = (exporter, xml_file)
return self.year_to_exporter[year][0] return self.year_to_exporter[year][0]
def process_item(self, item, spider): def process_item(self, item):
exporter = self._exporter_for_item(item) exporter = self._exporter_for_item(item)
exporter.export_item(item) exporter.export_item(item)
return item return item

42
docs/topics/item-pipeline.rst
View File

@ -26,7 +26,7 @@ Writing your own item pipeline
Each item pipeline is a :ref:`component <topics-components>` that must Each item pipeline is a :ref:`component <topics-components>` that must
implement the following method: implement the following method:
.. method:: process_item(self, item, spider) .. method:: process_item(self, item)
This method is called for every item pipeline component. This method is called for every item pipeline component.
@ -42,25 +42,16 @@ implement the following method:
:param item: the scraped item :param item: the scraped item
:type item: :ref:`item object <item-types>` :type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.Spider` object
Additionally, they may also implement the following methods: Additionally, they may also implement the following methods:
.. method:: open_spider(self, spider) .. method:: open_spider(self)
This method is called when the spider is opened. This method is called when the spider is opened.
:param spider: the spider which was opened .. method:: close_spider(self)
:type spider: :class:`~scrapy.Spider` object
.. method:: close_spider(self, spider)
This method is called when the spider is closed. This method is called when the spider is closed.
:param spider: the spider which was closed
:type spider: :class:`~scrapy.Spider` object
Item pipeline example Item pipeline example
===================== =====================
@ -82,7 +73,7 @@ contain a price:
class PricePipeline: class PricePipeline:
vat_factor = 1.15 vat_factor = 1.15
def process_item(self, item, spider): def process_item(self, item):
adapter = ItemAdapter(item) adapter = ItemAdapter(item)
if adapter.get("price"): if adapter.get("price"):
if adapter.get("price_excludes_vat"): if adapter.get("price_excludes_vat"):
@ -107,13 +98,13 @@ format:
class JsonWriterPipeline: class JsonWriterPipeline:
def open_spider(self, spider): def open_spider(self):
self.file = open("items.jsonl", "w") self.file = open("items.jsonl", "w")
def close_spider(self, spider): def close_spider(self):
self.file.close() self.file.close()
def process_item(self, item, spider): def process_item(self, item):
line = json.dumps(ItemAdapter(item).asdict()) + "\n" line = json.dumps(ItemAdapter(item).asdict()) + "\n"
self.file.write(line) self.file.write(line)
return item return item
@ -153,14 +144,14 @@ The main point of this example is to show how to :ref:`get the crawler
mongo_db=crawler.settings.get("MONGO_DATABASE", "items"), mongo_db=crawler.settings.get("MONGO_DATABASE", "items"),
) )
def open_spider(self, spider): def open_spider(self):
self.client = pymongo.MongoClient(self.mongo_uri) self.client = pymongo.MongoClient(self.mongo_uri)
self.db = self.client[self.mongo_db] self.db = self.client[self.mongo_db]
def close_spider(self, spider): def close_spider(self):
self.client.close() self.client.close()
def process_item(self, item, spider): def process_item(self, item):
self.db[self.collection_name].insert_one(ItemAdapter(item).asdict()) self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
return item return item
@ -198,12 +189,19 @@ item.
SPLASH_URL = "http://localhost:8050/render.png?url={}" SPLASH_URL = "http://localhost:8050/render.png?url={}"
async def process_item(self, item, spider): def __init__(crawler):
self.crawler = crawler
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
async def process_item(self, item):
adapter = ItemAdapter(item) adapter = ItemAdapter(item)
encoded_item_url = quote(adapter["url"]) encoded_item_url = quote(adapter["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url) screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url, callback=NO_CALLBACK) request = scrapy.Request(screenshot_url, callback=NO_CALLBACK)
response = await spider.crawler.engine.download_async(request) response = await self.crawler.engine.download_async(request)
if response.status != 200: if response.status != 200:
# Error happened, return item. # Error happened, return item.
@ -238,7 +236,7 @@ returns multiples items with the same id:
def __init__(self): def __init__(self):
self.ids_seen = set() self.ids_seen = set()
def process_item(self, item, spider): def process_item(self, item):
adapter = ItemAdapter(item) adapter = ItemAdapter(item)
if adapter["id"] in self.ids_seen: if adapter["id"] in self.ids_seen:
raise DropItem(f"Item ID already seen: {adapter['id']}") raise DropItem(f"Item ID already seen: {adapter['id']}")

2
docs/topics/settings.rst
View File

@ -554,7 +554,7 @@ When writing an item pipeline, you can force a different log level by setting
class MyPipeline: class MyPipeline:
def process_item(self, item, spider): def process_item(self, item):
if not item.get("price"): if not item.get("price"):
raise DropItem("Missing price data", log_level="INFO") raise DropItem("Missing price data", log_level="INFO")
return item return item

17
docs/topics/spider-middleware.rst
View File

@ -94,7 +94,7 @@ one or more of these methods:
def process_start_requests(self, start, spider): def process_start_requests(self, start, spider):
yield from start yield from start
.. method:: process_spider_input(response, spider) .. method:: process_spider_input(response)
This method is called for each response that goes through the spider This method is called for each response that goes through the spider
middleware and into the spider, for processing. middleware and into the spider, for processing.
@ -116,11 +116,7 @@ one or more of these methods:
:param response: the response being processed :param response: the response being processed
:type response: :class:`~scrapy.http.Response` object :type response: :class:`~scrapy.http.Response` object
:param spider: the spider for which this response is intended .. method:: process_spider_output(response, result)
:type spider: :class:`~scrapy.Spider` object
.. method:: process_spider_output(response, result, spider)
This method is called with the results returned from the Spider, after This method is called with the results returned from the Spider, after
it has processed the response. it has processed the response.
@ -149,10 +145,7 @@ one or more of these methods:
:type result: an iterable of :class:`~scrapy.Request` objects and :type result: an iterable of :class:`~scrapy.Request` objects and
:ref:`item objects <topics-items>` :ref:`item objects <topics-items>`
:param spider: the spider whose result is being processed .. method:: process_spider_output_async(response, result)
:type spider: :class:`~scrapy.Spider` object
.. method:: process_spider_output_async(response, result, spider)
:async: :async:
.. versionadded:: 2.7 .. versionadded:: 2.7
@ -161,7 +154,7 @@ one or more of these methods:
which will be called instead of :meth:`process_spider_output` if which will be called instead of :meth:`process_spider_output` if
``result`` is an :term:`asynchronous iterable`. ``result`` is an :term:`asynchronous iterable`.
.. method:: process_spider_exception(response, exception, spider) .. method:: process_spider_exception(response, exception)
This method is called when a spider or :meth:`process_spider_output` This method is called when a spider or :meth:`process_spider_output`
method (from a previous spider middleware) raises an exception. method (from a previous spider middleware) raises an exception.
@ -186,8 +179,6 @@ one or more of these methods:
:param exception: the exception raised :param exception: the exception raised
:type exception: :exc:`Exception` object :type exception: :exc:`Exception` object
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.Spider` object
Base class for custom spider middlewares Base class for custom spider middlewares
---------------------------------------- ----------------------------------------

33
scrapy/core/spidermw.py
View File

@ -117,9 +117,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
) )
def _add_middleware(self, mw: Any) -> None: def _add_middleware(self, mw: Any) -> None:
super()._add_middleware(mw)
if hasattr(mw, "process_spider_input"): if hasattr(mw, "process_spider_input"):
self.methods["process_spider_input"].append(mw.process_spider_input) self.methods["process_spider_input"].append(mw.process_spider_input)
self._check_mw_method_spider_arg(mw.process_spider_input)
if self._use_start_requests: if self._use_start_requests:
if hasattr(mw, "process_start_requests"): if hasattr(mw, "process_start_requests"):
self.methods["process_start_requests"].appendleft( self.methods["process_start_requests"].appendleft(
@ -127,10 +128,19 @@ class SpiderMiddlewareManager(MiddlewareManager):
) )
elif hasattr(mw, "process_start"): elif hasattr(mw, "process_start"):
self.methods["process_start"].appendleft(mw.process_start) self.methods["process_start"].appendleft(mw.process_start)
process_spider_output = self._get_async_method_pair(mw, "process_spider_output") process_spider_output = self._get_async_method_pair(mw, "process_spider_output")
self.methods["process_spider_output"].appendleft(process_spider_output) self.methods["process_spider_output"].appendleft(process_spider_output)
if callable(process_spider_output):
self._check_mw_method_spider_arg(process_spider_output)
elif isinstance(process_spider_output, tuple):
for m in process_spider_output:
self._check_mw_method_spider_arg(m)
process_spider_exception = getattr(mw, "process_spider_exception", None) process_spider_exception = getattr(mw, "process_spider_exception", None)
self.methods["process_spider_exception"].appendleft(process_spider_exception) self.methods["process_spider_exception"].appendleft(process_spider_exception)
if process_spider_exception is not None:
self._check_mw_method_spider_arg(process_spider_exception)
async def _process_spider_input( async def _process_spider_input(
self, self,
@ -141,7 +151,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
for method in self.methods["process_spider_input"]: for method in self.methods["process_spider_input"]:
method = cast("Callable", method) method = cast("Callable", method)
try: try:
result = method(response=response, spider=self._spider) if method in self._mw_methods_requiring_spider:
result = method(response=response, spider=self._spider)
else:
result = method(response=response)
if result is not None: if result is not None:
msg = ( msg = (
f"{global_object_name(method)} must return None " f"{global_object_name(method)} must return None "
@ -212,7 +225,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
if method is None: if method is None:
continue continue
method = cast("Callable", method) method = cast("Callable", method)
result = method(response=response, exception=exception, spider=self._spider) if method in self._mw_methods_requiring_spider:
result = method(
response=response, exception=exception, spider=self._spider
)
else:
result = method(response=response, exception=exception)
if _isiterable(result): if _isiterable(result):
# stop exception handling by handing control over to the # stop exception handling by handing control over to the
# process_spider_output chain if an iterable has been returned # process_spider_output chain if an iterable has been returned
@ -298,7 +316,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
) )
recovered = MutableChain(recovered_collected) recovered = MutableChain(recovered_collected)
# might fail directly if the output value is not a generator # might fail directly if the output value is not a generator
result = method(response=response, result=result, spider=self._spider) if method in self._mw_methods_requiring_spider:
result = method(
response=response, result=result, spider=self._spider
)
else:
result = method(response=response, result=result)
except Exception as ex: except Exception as ex:
exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = ( exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = (
self._process_spider_exception(response, ex, method_index + 1) self._process_spider_exception(response, ex, method_index + 1)
@ -421,7 +444,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
if self._use_start_requests: if self._use_start_requests:
sync_start = iter(self._spider.start_requests()) sync_start = iter(self._spider.start_requests())
sync_start = await self._process_chain( sync_start = await self._process_chain(
"process_start_requests", sync_start, self._spider "process_start_requests", sync_start, always_add_spider=True
) )
start: AsyncIterator[Any] = as_async_generator(sync_start) start: AsyncIterator[Any] = as_async_generator(sync_start)
else: else:

72
scrapy/middleware.py
View File

@ -8,7 +8,8 @@ from collections import defaultdict, deque
from typing import TYPE_CHECKING, Any, TypeVar, cast from typing import TYPE_CHECKING, Any, TypeVar, cast
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.utils.defer import _process_chain, process_parallel from scrapy.utils.defer import ensure_awaitable
from scrapy.utils.deprecate import argument_is_required
from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import global_object_name from scrapy.utils.python import global_object_name
@ -31,7 +32,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_T = TypeVar("_T") _T = TypeVar("_T")
_T2 = TypeVar("_T2")
class MiddlewareManager(ABC): class MiddlewareManager(ABC):
@ -56,6 +56,7 @@ class MiddlewareManager(ABC):
self.methods: dict[str, deque[Callable | tuple[Callable, Callable] | None]] = ( self.methods: dict[str, deque[Callable | tuple[Callable, Callable] | None]] = (
defaultdict(deque) defaultdict(deque)
) )
self._mw_methods_requiring_spider: set[Callable] = set()
for mw in middlewares: for mw in middlewares:
self._add_middleware(mw) self._add_middleware(mw)
@ -158,34 +159,53 @@ class MiddlewareManager(ABC):
) )
return cls(*middlewares, crawler=crawler) return cls(*middlewares, crawler=crawler)
def _add_middleware(self, mw: Any) -> None: def _add_middleware(self, mw: Any) -> None: # noqa: B027
if hasattr(mw, "open_spider"): pass
self.methods["open_spider"].append(mw.open_spider)
if hasattr(mw, "close_spider"):
self.methods["close_spider"].appendleft(mw.close_spider)
def _process_parallel( def _check_mw_method_spider_arg(self, method: Callable) -> None:
self, methodname: str, obj: _T, *args: Any if argument_is_required(method, "spider"):
) -> Deferred[list[_T2]]: warnings.warn(
methods = cast( f"{method.__qualname__}() requires a spider argument,"
"Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname] f" this is deprecated and the argument will not be passed in future Scrapy versions."
) f" If you need to access the spider instance you can save the crawler instance"
return process_parallel(methods, obj, *args) f" passed to from_crawler() and use its spider attribute.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
self._mw_methods_requiring_spider.add(method)
async def _process_chain(self, methodname: str, obj: _T, *args: Any) -> _T: async def _process_chain(
self,
methodname: str,
obj: _T,
*args: Any,
add_spider: bool = False,
always_add_spider: bool = False,
) -> _T:
methods = cast( methods = cast(
"Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname] "Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname]
) )
return await _process_chain(methods, obj, *args) for method in methods:
if always_add_spider or (
add_spider and method in self._mw_methods_requiring_spider
):
obj = await ensure_awaitable(method(obj, *(*args, self._spider)))
else:
obj = await ensure_awaitable(method(obj, *args))
return obj
def open_spider(self, spider: Spider | None = None) -> Deferred[list[None]]: def open_spider(
if spider: self, spider: Spider | None = None
self._warn_spider_arg("open_spider") ) -> Deferred[list[None]]: # pragma: no cover
self._set_compat_spider(spider) raise NotImplementedError(
return self._process_parallel("open_spider", self._spider) "MiddlewareManager.open_spider() is no longer implemented"
" and will be removed in a future Scrapy version."
)
def close_spider(self, spider: Spider | None = None) -> Deferred[list[None]]: def close_spider(
if spider: self, spider: Spider | None = None
self._warn_spider_arg("close_spider") ) -> Deferred[list[None]]: # pragma: no cover
self._set_compat_spider(spider) raise NotImplementedError(
return self._process_parallel("close_spider", self._spider) "MiddlewareManager.close_spider() is no longer implemented"
" and will be removed in a future Scrapy version."
)

52
scrapy/pipelines/__init__.py
View File

@ -7,16 +7,20 @@ See documentation in docs/item-pipeline.rst
from __future__ import annotations from __future__ import annotations
import warnings import warnings
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any, Callable, cast
from twisted.internet.defer import Deferred, DeferredList
from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.middleware import MiddlewareManager from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import deferred_from_coro from scrapy.utils.defer import deferred_from_coro, maybeDeferred_coro
from scrapy.utils.python import global_object_name from scrapy.utils.python import global_object_name
if TYPE_CHECKING: if TYPE_CHECKING:
from twisted.internet.defer import Deferred from collections.abc import Iterable
from twisted.python.failure import Failure
from scrapy import Spider from scrapy import Spider
from scrapy.settings import Settings from scrapy.settings import Settings
@ -30,9 +34,15 @@ class ItemPipelineManager(MiddlewareManager):
return build_component_list(settings.getwithbase("ITEM_PIPELINES")) return build_component_list(settings.getwithbase("ITEM_PIPELINES"))
def _add_middleware(self, pipe: Any) -> None: def _add_middleware(self, pipe: Any) -> None:
super()._add_middleware(pipe) if hasattr(pipe, "open_spider"):
self.methods["open_spider"].append(pipe.open_spider)
self._check_mw_method_spider_arg(pipe.open_spider)
if hasattr(pipe, "close_spider"):
self.methods["close_spider"].appendleft(pipe.close_spider)
self._check_mw_method_spider_arg(pipe.close_spider)
if hasattr(pipe, "process_item"): if hasattr(pipe, "process_item"):
self.methods["process_item"].append(pipe.process_item) self.methods["process_item"].append(pipe.process_item)
self._check_mw_method_spider_arg(pipe.process_item)
def process_item(self, item: Any, spider: Spider | None = None) -> Deferred[Any]: def process_item(self, item: Any, spider: Spider | None = None) -> Deferred[Any]:
if spider: if spider:
@ -45,4 +55,36 @@ class ItemPipelineManager(MiddlewareManager):
return deferred_from_coro(self.process_item_async(item)) return deferred_from_coro(self.process_item_async(item))
async def process_item_async(self, item: Any) -> Any: async def process_item_async(self, item: Any) -> Any:
return await self._process_chain("process_item", item, self._spider) return await self._process_chain("process_item", item, add_spider=True)
def _process_parallel(self, methodname: str) -> Deferred[list[None]]:
methods = cast("Iterable[Callable[..., None]]", self.methods[methodname])
def get_dfd(method: Callable[..., None]) -> Deferred[None]:
if method in self._mw_methods_requiring_spider:
return maybeDeferred_coro(method, self._spider)
return maybeDeferred_coro(method)
dfds = [get_dfd(m) for m in methods]
d: Deferred[list[tuple[bool, None]]] = DeferredList(
dfds, fireOnOneErrback=True, consumeErrors=True
)
d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r])
def eb(failure: Failure) -> Failure:
return failure.value.subFailure
d2.addErrback(eb)
return d2
def open_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:
if spider:
self._warn_spider_arg("open_spider")
self._set_compat_spider(spider)
return self._process_parallel("open_spider")
def close_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:
if spider:
self._warn_spider_arg("close_spider")
self._set_compat_spider(spider)
return self._process_parallel("close_spider")

21
scrapy/pipelines/media.py
View File

@ -165,12 +165,27 @@ class MediaPipeline(ABC):
pipe._finish_init(crawler) pipe._finish_init(crawler)
return pipe return pipe
def open_spider(self, spider: Spider) -> None: def open_spider(self, spider: Spider | None = None) -> None:
self.spiderinfo = self.SpiderInfo(spider) if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to MediaPipeline.open_spider()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
assert self.crawler.spider
self.spiderinfo = self.SpiderInfo(self.crawler.spider)
def process_item( def process_item(
self, item: Any, spider: Spider self, item: Any, spider: Spider | None = None
) -> Deferred[list[FileInfoOrError]]: ) -> Deferred[list[FileInfoOrError]]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to MediaPipeline.process_item()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
info = self.spiderinfo info = self.spiderinfo
requests = arg_to_iter(self.get_media_requests(item, info)) requests = arg_to_iter(self.get_media_requests(item, info))
dlist = [self._process_request(r, info, item) for r in requests] dlist = [self._process_request(r, info, item) for r in requests]

23
scrapy/spidermiddlewares/base.py
View File

@ -1,8 +1,10 @@
from __future__ import annotations from __future__ import annotations
import warnings
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider from scrapy import Request, Spider
from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterable from collections.abc import AsyncIterator, Iterable
@ -53,15 +55,32 @@ class BaseSpiderMiddleware:
yield o yield o
def process_spider_output( def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider self, response: Response, result: Iterable[Any], spider: Spider | None = None
) -> Iterable[Any]: ) -> Iterable[Any]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to BaseSpiderMiddleware.process_spider_output()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
for o in result: for o in result:
if (o := self._get_processed(o, response)) is not None: if (o := self._get_processed(o, response)) is not None:
yield o yield o
async def process_spider_output_async( async def process_spider_output_async(
self, response: Response, result: AsyncIterator[Any], spider: Spider self,
response: Response,
result: AsyncIterator[Any],
spider: Spider | None = None,
) -> AsyncIterator[Any]: ) -> AsyncIterator[Any]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to BaseSpiderMiddleware.process_spider_output_async()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
async for o in result: async for o in result:
if (o := self._get_processed(o, response)) is not None: if (o := self._get_processed(o, response)) is not None:
yield o yield o

27
scrapy/spidermiddlewares/depth.py
View File

@ -7,8 +7,10 @@ See documentation in docs/topics/spider-middleware.rst
from __future__ import annotations from __future__ import annotations
import logging import logging
import warnings
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
if TYPE_CHECKING: if TYPE_CHECKING:
@ -53,16 +55,33 @@ class DepthMiddleware(BaseSpiderMiddleware):
return o return o
def process_spider_output( def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider self, response: Response, result: Iterable[Any], spider: Spider | None = None
) -> Iterable[Any]: ) -> Iterable[Any]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to DepthMiddleware.process_spider_output()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
self._init_depth(response) self._init_depth(response)
yield from super().process_spider_output(response, result, spider) yield from super().process_spider_output(response, result)
async def process_spider_output_async( async def process_spider_output_async(
self, response: Response, result: AsyncIterator[Any], spider: Spider self,
response: Response,
result: AsyncIterator[Any],
spider: Spider | None = None,
) -> AsyncIterator[Any]: ) -> AsyncIterator[Any]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to DepthMiddleware.process_spider_output_async()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
self._init_depth(response) self._init_depth(response)
async for o in super().process_spider_output_async(response, result, spider): async for o in super().process_spider_output_async(response, result):
yield o yield o
def _init_depth(self, response: Response) -> None: def _init_depth(self, response: Response) -> None:

45
scrapy/spidermiddlewares/httperror.py
View File

@ -7,9 +7,10 @@ See documentation in docs/topics/spider-middleware.rst
from __future__ import annotations from __future__ import annotations
import logging import logging
import warnings
from typing import TYPE_CHECKING, Any from typing import TYPE_CHECKING, Any
from scrapy.exceptions import IgnoreRequest from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
if TYPE_CHECKING: if TYPE_CHECKING:
from collections.abc import Iterable from collections.abc import Iterable
@ -35,9 +36,7 @@ class HttpError(IgnoreRequest):
class HttpErrorMiddleware: class HttpErrorMiddleware:
@classmethod crawler: Crawler
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
def __init__(self, settings: BaseSettings): def __init__(self, settings: BaseSettings):
self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL") self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL")
@ -45,7 +44,22 @@ class HttpErrorMiddleware:
"HTTPERROR_ALLOWED_CODES" "HTTPERROR_ALLOWED_CODES"
) )
def process_spider_input(self, response: Response, spider: Spider) -> None: @classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
o = cls(crawler.settings)
o.crawler = crawler
return o
def process_spider_input(
self, response: Response, spider: Spider | None = None
) -> None:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to HttpErrorMiddleware.process_spider_input()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
if 200 <= response.status < 300: # common case if 200 <= response.status < 300: # common case
return return
meta = response.meta meta = response.meta
@ -57,25 +71,34 @@ class HttpErrorMiddleware:
return return
else: else:
allowed_statuses = getattr( allowed_statuses = getattr(
spider, "handle_httpstatus_list", self.handle_httpstatus_list self.crawler.spider,
"handle_httpstatus_list",
self.handle_httpstatus_list,
) )
if response.status in allowed_statuses: if response.status in allowed_statuses:
return return
raise HttpError(response, "Ignoring non-200 response") raise HttpError(response, "Ignoring non-200 response")
def process_spider_exception( def process_spider_exception(
self, response: Response, exception: Exception, spider: Spider self, response: Response, exception: Exception, spider: Spider | None = None
) -> Iterable[Any] | None: ) -> Iterable[Any] | None:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to HttpErrorMiddleware.process_spider_exception()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
if isinstance(exception, HttpError): if isinstance(exception, HttpError):
assert spider.crawler.stats assert self.crawler.stats
spider.crawler.stats.inc_value("httperror/response_ignored_count") self.crawler.stats.inc_value("httperror/response_ignored_count")
spider.crawler.stats.inc_value( self.crawler.stats.inc_value(
f"httperror/response_ignored_status_count/{response.status}" f"httperror/response_ignored_status_count/{response.status}"
) )
logger.info( logger.info(
"Ignoring response %(response)r: HTTP status code is not handled or not allowed", "Ignoring response %(response)r: HTTP status code is not handled or not allowed",
{"response": response}, {"response": response},
extra={"spider": spider}, extra={"spider": self.crawler.spider},
) )
return [] return []
return None return None

20
scrapy/utils/defer.py
View File

@ -312,19 +312,6 @@ def process_chain(
return d return d
async def _process_chain(
callables: Iterable[Callable[Concatenate[_T, _P], _T | Awaitable[_T]]],
input_: _T,
*a: _P.args,
**kw: _P.kwargs,
) -> _T:
"""Chain the given (potentialy asynchronous) callables."""
result = input_
for callable_ in callables:
result = await ensure_awaitable(callable_(result, *a, **kw))
return result
def process_chain_both( def process_chain_both(
callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]], errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]],
@ -355,10 +342,15 @@ def process_parallel(
input: _T, # noqa: A002 input: _T, # noqa: A002
*a: _P.args, *a: _P.args,
**kw: _P.kwargs, **kw: _P.kwargs,
) -> Deferred[list[_T2]]: ) -> Deferred[list[_T2]]: # pragma: no cover
"""Return a Deferred with the output of all successful calls to the given """Return a Deferred with the output of all successful calls to the given
callbacks callbacks
""" """
warnings.warn(
"process_parallel() is deprecated.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
dfds = [succeed(input).addCallback(x, *a, **kw) for x in callbacks] dfds = [succeed(input).addCallback(x, *a, **kw) for x in callbacks]
d: Deferred[list[tuple[bool, _T2]]] = DeferredList( d: Deferred[list[tuple[bool, _T2]]] = DeferredList(
dfds, fireOnOneErrback=True, consumeErrors=True dfds, fireOnOneErrback=True, consumeErrors=True

2
tests/AsyncCrawlerProcess/asyncio_deferred_signal.py
View File

@ -16,7 +16,7 @@ class UppercasePipeline:
def open_spider(self, spider): def open_spider(self, spider):
return deferred_from_coro(self._open_spider(spider)) return deferred_from_coro(self._open_spider(spider))
def process_item(self, item, spider): def process_item(self, item):
return {"url": item["url"].upper()} return {"url": item["url"].upper()}

2
tests/CrawlerProcess/asyncio_deferred_signal.py
View File

@ -16,7 +16,7 @@ class UppercasePipeline:
def open_spider(self, spider): def open_spider(self, spider):
return deferred_from_coro(self._open_spider(spider)) return deferred_from_coro(self._open_spider(spider))
def process_item(self, item, spider): def process_item(self, item):
return {"url": item["url"].upper()} return {"url": item["url"].upper()}

6
tests/pipelines.py
View File

@ -4,13 +4,13 @@ Some pipelines used for testing
class ZeroDivisionErrorPipeline: class ZeroDivisionErrorPipeline:
def open_spider(self, spider): def open_spider(self):
1 / 0 1 / 0
def process_item(self, item, spider): def process_item(self, item):
return item return item
class ProcessWithZeroDivisionErrorPipeline: class ProcessWithZeroDivisionErrorPipeline:
def process_item(self, item, spider): def process_item(self, item):
1 / 0 1 / 0

4
tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
View File

@ -2,7 +2,7 @@ class TestSpiderPipeline:
def open_spider(self, spider): def open_spider(self, spider):
pass pass
def process_item(self, item, spider): def process_item(self, item):
return item return item
@ -10,5 +10,5 @@ class TestSpiderExceptionPipeline:
def open_spider(self, spider): def open_spider(self, spider):
raise RuntimeError("exception") raise RuntimeError("exception")
def process_item(self, item, spider): def process_item(self, item):
return item return item

2
tests/test_command_parse.py
View File

@ -153,7 +153,7 @@ import logging
class MyPipeline: class MyPipeline:
component_name = 'my_pipeline' component_name = 'my_pipeline'
def process_item(self, item, spider): def process_item(self, item):
logging.info('It Works!') logging.info('It Works!')
return item return item
""", """,

2
tests/test_logformatter.py
View File

@ -246,7 +246,7 @@ class SkipMessagesLogFormatter(LogFormatter):
class DropSomeItemsPipeline: class DropSomeItemsPipeline:
drop = True drop = True
def process_item(self, item, spider): def process_item(self, item):
if self.drop: if self.drop:
self.drop = False self.drop = False
raise DropItem("Ignoring item") raise DropItem("Ignoring item")

76
tests/test_middleware.py
View File

@ -7,7 +7,6 @@ import pytest
from scrapy import Spider from scrapy import Spider
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.middleware import MiddlewareManager from scrapy.middleware import MiddlewareManager
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
if TYPE_CHECKING: if TYPE_CHECKING:
@ -57,7 +56,10 @@ class MyMiddlewareManager(MiddlewareManager):
return [M1, MOff, M3] return [M1, MOff, M3]
def _add_middleware(self, mw): def _add_middleware(self, mw):
super()._add_middleware(mw) if hasattr(mw, "open_spider"):
self.methods["open_spider"].append(mw.open_spider)
if hasattr(mw, "close_spider"):
self.methods["close_spider"].appendleft(mw.close_spider)
if hasattr(mw, "process"): if hasattr(mw, "process"):
self.methods["process"].append(mw.process) self.methods["process"].append(mw.process)
@ -105,73 +107,3 @@ def test_no_crawler() -> None:
mwman = MyMiddlewareManager(m1, m2, m3) mwman = MyMiddlewareManager(m1, m2, m3)
assert mwman.middlewares == (m1, m2, m3) assert mwman.middlewares == (m1, m2, m3)
assert mwman.crawler is None assert mwman.crawler is None
def test_deprecated_spider_arg_no_crawler_spider(crawler: Crawler) -> None:
"""Crawler is provided, but doesn't have a spider. The instance passed to the method is
ignored and raises a warning."""
mwman = MyMiddlewareManager(crawler=crawler)
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated",
),
pytest.raises(
ValueError,
match="MyMiddlewareManager needs to access self.crawler.spider but it is None",
),
):
mwman.open_spider(DefaultSpider())
def test_deprecated_spider_arg_with_crawler(crawler: Crawler) -> None:
"""Crawler is provided and has a spider, works. The instance passed to the method is ignored,
even if mismatched, but raises a warning."""
mwman = MyMiddlewareManager(crawler=crawler)
crawler.spider = crawler._create_spider("foo")
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated",
):
mwman.open_spider(DefaultSpider())
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to MyMiddlewareManager.close_spider\(\) is deprecated",
):
mwman.close_spider(DefaultSpider())
def test_deprecated_spider_arg_without_crawler() -> None:
"""The first instance passed to the method is used, with a warning. Mismatched ones raise an error."""
with pytest.warns(
ScrapyDeprecationWarning,
match="was called without the crawler argument",
):
mwman = MyMiddlewareManager()
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated",
):
mwman.open_spider(DefaultSpider())
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to MyMiddlewareManager.close_spider\(\) is deprecated",
),
pytest.raises(RuntimeError, match="Different instances of Spider were passed"),
):
mwman.close_spider(DefaultSpider())
def test_no_spider_arg_without_crawler() -> None:
"""If no crawler and no spider arg, raise an error."""
with pytest.warns(
ScrapyDeprecationWarning,
match="was called without the crawler argument",
):
mwman = MyMiddlewareManager()
with pytest.raises(
ValueError,
match="has no known Spider instance",
):
mwman.open_spider()

6
tests/test_pipeline_files.py
View File

@ -29,6 +29,7 @@ from scrapy.pipelines.files import (
GCSFilesStore, GCSFilesStore,
S3FilesStore, S3FilesStore,
) )
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer from tests.mockserver.ftp import MockFTPServer
@ -78,10 +79,11 @@ class TestFilesPipeline:
def setup_method(self): def setup_method(self):
self.tempdir = mkdtemp() self.tempdir = mkdtemp()
settings_dict = {"FILES_STORE": self.tempdir} settings_dict = {"FILES_STORE": self.tempdir}
crawler = get_crawler(spidercls=None, settings_dict=settings_dict) crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
crawler.spider = crawler._create_spider()
self.pipeline = FilesPipeline.from_crawler(crawler) self.pipeline = FilesPipeline.from_crawler(crawler)
self.pipeline.download_func = _mocked_download_func self.pipeline.download_func = _mocked_download_func
self.pipeline.open_spider(None) self.pipeline.open_spider()
def teardown_method(self): def teardown_method(self):
rmtree(self.tempdir) rmtree(self.tempdir)

40
tests/test_pipeline_media.py
View File

@ -13,10 +13,10 @@ from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FileException from scrapy.pipelines.files import FileException
from scrapy.pipelines.media import MediaPipeline from scrapy.pipelines.media import MediaPipeline
from scrapy.spiders import Spider
from scrapy.utils.asyncio import call_later from scrapy.utils.asyncio import call_later
from scrapy.utils.log import failure_to_exc_info from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.signal import disconnect_all from scrapy.utils.signal import disconnect_all
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
@ -48,12 +48,11 @@ class TestBaseMediaPipeline:
settings = None settings = None
def setup_method(self): def setup_method(self):
spider_cls = Spider crawler = get_crawler(DefaultSpider, self.settings)
self.spider = spider_cls("media.com") crawler.spider = crawler._create_spider()
crawler = get_crawler(spider_cls, self.settings)
self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe = self.pipeline_class.from_crawler(crawler)
self.pipe.download_func = _mocked_download_func self.pipe.download_func = _mocked_download_func
self.pipe.open_spider(self.spider) self.pipe.open_spider()
self.info = self.pipe.spiderinfo self.info = self.pipe.spiderinfo
self.fingerprint = crawler.request_fingerprinter.fingerprint self.fingerprint = crawler.request_fingerprinter.fingerprint
@ -162,7 +161,7 @@ class TestBaseMediaPipeline:
@inlineCallbacks @inlineCallbacks
def test_default_process_item(self): def test_default_process_item(self):
item = {"name": "name"} item = {"name": "name"}
new_item = yield self.pipe.process_item(item, self.spider) new_item = yield self.pipe.process_item(item)
assert new_item is item assert new_item is item
@ -216,7 +215,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
errback=self._errback, errback=self._errback,
) )
item = {"requests": req} item = {"requests": req}
new_item = yield self.pipe.process_item(item, self.spider) new_item = yield self.pipe.process_item(item)
assert new_item["results"] == [(True, {})] assert new_item["results"] == [(True, {})]
assert self.pipe._mockcalled == [ assert self.pipe._mockcalled == [
"get_media_requests", "get_media_requests",
@ -236,7 +235,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
errback=self._errback, errback=self._errback,
) )
item = {"requests": req} item = {"requests": req}
new_item = yield self.pipe.process_item(item, self.spider) new_item = yield self.pipe.process_item(item)
assert len(new_item["results"]) == 1 assert len(new_item["results"]) == 1
assert new_item["results"][0][0] is False assert new_item["results"][0][0] is False
assert isinstance(new_item["results"][0][1], Failure) assert isinstance(new_item["results"][0][1], Failure)
@ -258,7 +257,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
fail = Failure(exc) fail = Failure(exc)
req2 = Request("http://url2", meta={"response": fail}) req2 = Request("http://url2", meta={"response": fail})
item = {"requests": [req1, req2]} item = {"requests": [req1, req2]}
new_item = yield self.pipe.process_item(item, self.spider) new_item = yield self.pipe.process_item(item)
assert len(new_item["results"]) == 2 assert len(new_item["results"]) == 2
assert new_item["results"][0] == (True, {}) assert new_item["results"][0] == (True, {})
assert new_item["results"][1][0] is False assert new_item["results"][1][0] is False
@ -281,7 +280,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
# returns single Request (without callback) # returns single Request (without callback)
req = Request("http://url") req = Request("http://url")
item = {"requests": req} # pass a single item item = {"requests": req} # pass a single item
new_item = yield self.pipe.process_item(item, self.spider) new_item = yield self.pipe.process_item(item)
assert new_item is item assert new_item is item
assert self.fingerprint(req) in self.info.downloaded assert self.fingerprint(req) in self.info.downloaded
@ -289,7 +288,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
req1 = Request("http://url1") req1 = Request("http://url1")
req2 = Request("http://url2") req2 = Request("http://url2")
item = {"requests": iter([req1, req2])} item = {"requests": iter([req1, req2])}
new_item = yield self.pipe.process_item(item, self.spider) new_item = yield self.pipe.process_item(item)
assert new_item is item assert new_item is item
assert self.fingerprint(req1) in self.info.downloaded assert self.fingerprint(req1) in self.info.downloaded
assert self.fingerprint(req2) in self.info.downloaded assert self.fingerprint(req2) in self.info.downloaded
@ -299,7 +298,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
rsp1 = Response("http://url1") rsp1 = Response("http://url1")
req1 = Request("http://url1", meta={"response": rsp1}) req1 = Request("http://url1", meta={"response": rsp1})
item = {"requests": req1} item = {"requests": req1}
new_item = yield self.pipe.process_item(item, self.spider) new_item = yield self.pipe.process_item(item)
assert new_item is item assert new_item is item
assert new_item["results"] == [(True, {})] assert new_item["results"] == [(True, {})]
@ -308,7 +307,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
req1.url, meta={"response": Response("http://donot.download.me")} req1.url, meta={"response": Response("http://donot.download.me")}
) )
item = {"requests": req2} item = {"requests": req2}
new_item = yield self.pipe.process_item(item, self.spider) new_item = yield self.pipe.process_item(item)
assert new_item is item assert new_item is item
assert self.fingerprint(req1) == self.fingerprint(req2) assert self.fingerprint(req1) == self.fingerprint(req2)
assert new_item["results"] == [(True, {})] assert new_item["results"] == [(True, {})]
@ -321,7 +320,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
req1.url, meta={"response": Response("http://donot.download.me")} req1.url, meta={"response": Response("http://donot.download.me")}
) )
item = {"requests": [req1, req2]} item = {"requests": [req1, req2]}
new_item = yield self.pipe.process_item(item, self.spider) new_item = yield self.pipe.process_item(item)
assert new_item is item assert new_item is item
assert new_item["results"] == [(True, {}), (True, {})] assert new_item["results"] == [(True, {}), (True, {})]
@ -348,14 +347,14 @@ class TestMediaPipeline(TestBaseMediaPipeline):
req1 = Request("http://url", meta={"response": rsp1_func}) req1 = Request("http://url", meta={"response": rsp1_func})
req2 = Request(req1.url, meta={"response": rsp2_func}) req2 = Request(req1.url, meta={"response": rsp2_func})
item = {"requests": [req1, req2]} item = {"requests": [req1, req2]}
new_item = yield self.pipe.process_item(item, self.spider) new_item = yield self.pipe.process_item(item)
assert new_item["results"] == [(True, {}), (True, {})] assert new_item["results"] == [(True, {}), (True, {})]
@inlineCallbacks @inlineCallbacks
def test_use_media_to_download_result(self): def test_use_media_to_download_result(self):
req = Request("http://url", meta={"result": "ITSME"}) req = Request("http://url", meta={"result": "ITSME"})
item = {"requests": req} item = {"requests": req}
new_item = yield self.pipe.process_item(item, self.spider) new_item = yield self.pipe.process_item(item)
assert new_item["results"] == [(True, "ITSME")] assert new_item["results"] == [(True, "ITSME")]
assert self.pipe._mockcalled == [ assert self.pipe._mockcalled == [
"get_media_requests", "get_media_requests",
@ -555,12 +554,11 @@ class TestMediaFailedFailure:
settings = None settings = None
def setup_method(self): def setup_method(self):
spider_cls = Spider crawler = get_crawler(DefaultSpider, self.settings)
self.spider = spider_cls("media.com") crawler.spider = crawler._create_spider()
crawler = get_crawler(spider_cls, self.settings)
self.pipe = self.pipeline_class.from_crawler(crawler) self.pipe = self.pipeline_class.from_crawler(crawler)
self.pipe.download_func = _mocked_download_func self.pipe.download_func = _mocked_download_func
self.pipe.open_spider(self.spider) self.pipe.open_spider()
self.info = self.pipe.spiderinfo self.info = self.pipe.spiderinfo
self.fingerprint = crawler.request_fingerprinter.fingerprint self.fingerprint = crawler.request_fingerprinter.fingerprint
@ -587,7 +585,7 @@ class TestMediaFailedFailure:
with pytest.warns( with pytest.warns(
ScrapyDeprecationWarning, match="media_failed returned a Failure instance" ScrapyDeprecationWarning, match="media_failed returned a Failure instance"
): ):
new_item = yield self.pipe.process_item(item, self.spider) new_item = yield self.pipe.process_item(item)
assert len(new_item["results"]) == 1 assert len(new_item["results"]) == 1
assert new_item["results"][0][0] is False assert new_item["results"][0][0] is False
assert isinstance(new_item["results"][0][1], Failure) assert isinstance(new_item["results"][0][1], Failure)

157
tests/test_pipelines.py
View File

@ -4,6 +4,7 @@ import pytest
from twisted.internet.defer import Deferred, inlineCallbacks, succeed from twisted.internet.defer import Deferred, inlineCallbacks, succeed
from scrapy import Request, Spider, signals from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.pipelines import ItemPipelineManager from scrapy.pipelines import ItemPipelineManager
from scrapy.utils.asyncio import call_later from scrapy.utils.asyncio import call_later
@ -19,17 +20,25 @@ from tests.mockserver.http import MockServer
class SimplePipeline: class SimplePipeline:
def process_item(self, item, spider): def process_item(self, item):
item["pipeline_passed"] = True item["pipeline_passed"] = True
return item return item
class DeprecatedSpiderArgPipeline:
def open_spider(self, spider):
pass
def close_spider(self, spider):
pass
class DeferredPipeline: class DeferredPipeline:
def cb(self, item): def cb(self, item):
item["pipeline_passed"] = True item["pipeline_passed"] = True
return item return item
def process_item(self, item, spider): def process_item(self, item):
d = Deferred() d = Deferred()
d.addCallback(self.cb) d.addCallback(self.cb)
d.callback(item) d.callback(item)
@ -37,7 +46,7 @@ class DeferredPipeline:
class AsyncDefPipeline: class AsyncDefPipeline:
async def process_item(self, item, spider): async def process_item(self, item):
d = Deferred() d = Deferred()
call_later(0, d.callback, None) call_later(0, d.callback, None)
await maybe_deferred_to_future(d) await maybe_deferred_to_future(d)
@ -46,7 +55,7 @@ class AsyncDefPipeline:
class AsyncDefAsyncioPipeline: class AsyncDefAsyncioPipeline:
async def process_item(self, item, spider): async def process_item(self, item):
d = Deferred() d = Deferred()
loop = asyncio.get_event_loop() loop = asyncio.get_event_loop()
loop.call_later(0, d.callback, None) loop.call_later(0, d.callback, None)
@ -57,7 +66,7 @@ class AsyncDefAsyncioPipeline:
class AsyncDefNotAsyncioPipeline: class AsyncDefNotAsyncioPipeline:
async def process_item(self, item, spider): async def process_item(self, item):
d1 = Deferred() d1 = Deferred()
from twisted.internet import reactor from twisted.internet import reactor
@ -184,6 +193,9 @@ class TestCustomPipelineManager:
def open_spider(self, spider): # pylint: disable=signature-differs def open_spider(self, spider): # pylint: disable=signature-differs
return super().open_spider(spider) return super().open_spider(spider)
def close_spider(self, spider): # pylint: disable=signature-differs
return super().close_spider(spider)
def process_item(self, item, spider): # pylint: disable=signature-differs def process_item(self, item, spider): # pylint: disable=signature-differs
with pytest.warns( with pytest.warns(
ScrapyDeprecationWarning, ScrapyDeprecationWarning,
@ -212,6 +224,10 @@ class TestCustomPipelineManager:
ScrapyDeprecationWarning, ScrapyDeprecationWarning,
match=r"The open_spider\(\) method of .+\.CustomPipelineManager requires a spider argument", match=r"The open_spider\(\) method of .+\.CustomPipelineManager requires a spider argument",
), ),
pytest.warns(
ScrapyDeprecationWarning,
match=r"The close_spider\(\) method of .+\.CustomPipelineManager requires a spider argument",
),
pytest.warns( pytest.warns(
ScrapyDeprecationWarning, ScrapyDeprecationWarning,
match=r"The process_item\(\) method of .+\.CustomPipelineManager requires a spider argument", match=r"The process_item\(\) method of .+\.CustomPipelineManager requires a spider argument",
@ -220,6 +236,10 @@ class TestCustomPipelineManager:
ScrapyDeprecationWarning, ScrapyDeprecationWarning,
match=r"Passing a spider argument to CustomPipelineManager.open_spider\(\) is deprecated", match=r"Passing a spider argument to CustomPipelineManager.open_spider\(\) is deprecated",
), ),
pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to CustomPipelineManager.close_spider\(\) is deprecated",
),
pytest.warns( pytest.warns(
ScrapyDeprecationWarning, ScrapyDeprecationWarning,
match=r"CustomPipelineManager overrides process_item\(\) but doesn't override process_item_async\(\)", match=r"CustomPipelineManager overrides process_item\(\) but doesn't override process_item_async\(\)",
@ -254,7 +274,7 @@ class TestCustomPipelineManager:
def process_item(self, item, spider): def process_item(self, item, spider):
for pipeline in self.pipelines: for pipeline in self.pipelines:
item = pipeline.process_item(item, spider) item = pipeline.process_item(item)
return succeed(item) return succeed(item)
items = [] items = []
@ -294,3 +314,128 @@ class TestCustomPipelineManager:
await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver)) await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
assert len(items) == 1 assert len(items) == 1
class TestMiddlewareManagerSpider:
"""Tests for the deprecated spider arg handling in MiddlewareManager.
Here because MiddlewareManager doesn't have methods that could take a spider arg."""
@pytest.fixture
def crawler(self) -> Crawler:
return get_crawler(Spider)
def test_deprecated_spider_arg_no_crawler_spider(self, crawler: Crawler) -> None:
"""Crawler is provided, but doesn't have a spider. The instance passed to the method is
ignored and raises a warning."""
mwman = ItemPipelineManager(crawler=crawler)
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",
),
):
mwman._add_middleware(DeprecatedSpiderArgPipeline())
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated",
),
pytest.raises(
ValueError,
match="ItemPipelineManager needs to access self.crawler.spider but it is None",
),
):
mwman.open_spider(DefaultSpider())
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated",
),
pytest.raises(
ValueError,
match="ItemPipelineManager needs to access self.crawler.spider but it is None",
),
):
mwman.close_spider(DefaultSpider())
def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None:
"""Crawler is provided and has a spider, works. The instance passed to the method is ignored,
even if mismatched, but raises a warning."""
mwman = ItemPipelineManager(crawler=crawler)
crawler.spider = crawler._create_spider("foo")
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated",
):
mwman.open_spider(DefaultSpider())
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated",
):
mwman.close_spider(DefaultSpider())
def test_deprecated_spider_arg_without_crawler(self) -> None:
"""The first instance passed to the method is used, with a warning. Mismatched ones raise an error."""
with pytest.warns(
ScrapyDeprecationWarning,
match="was called without the crawler argument",
):
mwman = ItemPipelineManager()
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",
),
):
mwman._add_middleware(DeprecatedSpiderArgPipeline())
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated",
):
mwman.open_spider(DefaultSpider())
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated",
),
pytest.raises(
RuntimeError, match="Different instances of Spider were passed"
),
):
mwman.close_spider(DefaultSpider())
mwman.close_spider()
def test_no_spider_arg_without_crawler(self) -> None:
"""If no crawler and no spider arg, raise an error."""
with pytest.warns(
ScrapyDeprecationWarning,
match="was called without the crawler argument",
):
mwman = ItemPipelineManager()
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",
),
):
mwman._add_middleware(DeprecatedSpiderArgPipeline())
with (
pytest.raises(
ValueError,
match="has no known Spider instance",
),
):
mwman.open_spider()

4
tests/test_request_cb_kwargs.py
View File

@ -33,12 +33,12 @@ class InjectArgumentsSpiderMiddleware:
request.cb_kwargs["from_process_start"] = True request.cb_kwargs["from_process_start"] = True
yield request yield request
def process_spider_input(self, response, spider): def process_spider_input(self, response):
request = response.request request = response.request
if request.callback.__name__ == "parse_spider_mw": if request.callback.__name__ == "parse_spider_mw":
request.cb_kwargs["from_process_spider_input"] = True request.cb_kwargs["from_process_spider_input"] = True
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
for element in result: for element in result:
if ( if (
isinstance(element, Request) isinstance(element, Request)

4
tests/test_settings/__init__.py
View File

@ -451,7 +451,7 @@ class TestSettings:
def test_passing_objects_as_values(self): def test_passing_objects_as_values(self):
class TestPipeline: class TestPipeline:
def process_item(self, i, s): def process_item(self, i):
return i return i
settings = Settings( settings = Settings(
@ -471,7 +471,7 @@ class TestSettings:
assert priority == 800 assert priority == 800
assert mypipeline == TestPipeline assert mypipeline == TestPipeline
assert isinstance(mypipeline(), TestPipeline) assert isinstance(mypipeline(), TestPipeline)
assert mypipeline().process_item("item", None) == "item" assert mypipeline().process_item("item") == "item"
myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp") myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp")
assert myhandler == FileDownloadHandler assert myhandler == FileDownloadHandler

71
tests/test_spidermiddleware.py
View File

@ -10,7 +10,7 @@ from testfixtures import LogCapture
from twisted.internet import defer from twisted.internet import defer
from scrapy.core.spidermw import SpiderMiddlewareManager from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import _InvalidOutput from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
from scrapy.http import Request, Response from scrapy.http import Request, Response
from scrapy.spiders import Spider from scrapy.spiders import Spider
from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.asyncgen import collect_asyncgen
@ -54,7 +54,7 @@ class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
@deferred_f_from_coro_f @deferred_f_from_coro_f
async def test_invalid_process_spider_input(self): async def test_invalid_process_spider_input(self):
class InvalidProcessSpiderInputMiddleware: class InvalidProcessSpiderInputMiddleware:
def process_spider_input(self, response, spider): def process_spider_input(self, response):
return 1 return 1
self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware()) self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())
@ -68,7 +68,7 @@ class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
@deferred_f_from_coro_f @deferred_f_from_coro_f
async def test_invalid_process_spider_output(self): async def test_invalid_process_spider_output(self):
class InvalidProcessSpiderOutputMiddleware: class InvalidProcessSpiderOutputMiddleware:
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
return 1 return 1
self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())
@ -82,11 +82,11 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
@deferred_f_from_coro_f @deferred_f_from_coro_f
async def test_invalid_process_spider_exception(self): async def test_invalid_process_spider_exception(self):
class InvalidProcessSpiderOutputExceptionMiddleware: class InvalidProcessSpiderOutputExceptionMiddleware:
def process_spider_exception(self, response, exception, spider): def process_spider_exception(self, response, exception):
return 1 return 1
class RaiseExceptionProcessSpiderOutputMiddleware: class RaiseExceptionProcessSpiderOutputMiddleware:
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
raise RuntimeError raise RuntimeError
self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())
@ -101,11 +101,11 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
@deferred_f_from_coro_f @deferred_f_from_coro_f
async def test_process_spider_exception_return_none(self): async def test_process_spider_exception_return_none(self):
class ProcessSpiderExceptionReturnNoneMiddleware: class ProcessSpiderExceptionReturnNoneMiddleware:
def process_spider_exception(self, response, exception, spider): def process_spider_exception(self, response, exception):
return None return None
class RaiseExceptionProcessSpiderOutputMiddleware: class RaiseExceptionProcessSpiderOutputMiddleware:
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
1 / 0 1 / 0
self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
@ -191,34 +191,34 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
class ProcessSpiderOutputSimpleMiddleware: class ProcessSpiderOutputSimpleMiddleware:
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
yield from result yield from result
class ProcessSpiderOutputAsyncGenMiddleware: class ProcessSpiderOutputAsyncGenMiddleware:
async def process_spider_output(self, response, result, spider): async def process_spider_output(self, response, result):
async for r in result: async for r in result:
yield r yield r
class ProcessSpiderOutputUniversalMiddleware: class ProcessSpiderOutputUniversalMiddleware:
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
yield from result yield from result
async def process_spider_output_async(self, response, result, spider): async def process_spider_output_async(self, response, result):
async for r in result: async for r in result:
yield r yield r
class ProcessSpiderExceptionSimpleIterableMiddleware: class ProcessSpiderExceptionSimpleIterableMiddleware:
def process_spider_exception(self, response, exception, spider): def process_spider_exception(self, response, exception):
yield {"foo": 1} yield {"foo": 1}
yield {"foo": 2} yield {"foo": 2}
yield {"foo": 3} yield {"foo": 3}
class ProcessSpiderExceptionAsyncIteratorMiddleware: class ProcessSpiderExceptionAsyncIteratorMiddleware:
async def process_spider_exception(self, response, exception, spider): async def process_spider_exception(self, response, exception):
yield {"foo": 1} yield {"foo": 1}
d = defer.Deferred() d = defer.Deferred()
call_later(0, d.callback, None) call_later(0, d.callback, None)
@ -315,12 +315,12 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
class ProcessSpiderOutputNonIterableMiddleware: class ProcessSpiderOutputNonIterableMiddleware:
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
return return
class ProcessSpiderOutputCoroutineMiddleware: class ProcessSpiderOutputCoroutineMiddleware:
async def process_spider_output(self, response, result, spider): async def process_spider_output(self, response, result):
return result return result
@ -384,23 +384,23 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):
class UniversalMiddlewareNoSync: class UniversalMiddlewareNoSync:
async def process_spider_output_async(self, response, result, spider): async def process_spider_output_async(self, response, result):
yield yield
class UniversalMiddlewareBothSync: class UniversalMiddlewareBothSync:
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
yield yield
def process_spider_output_async(self, response, result, spider): def process_spider_output_async(self, response, result):
yield yield
class UniversalMiddlewareBothAsync: class UniversalMiddlewareBothAsync:
async def process_spider_output(self, response, result, spider): async def process_spider_output(self, response, result):
yield yield
async def process_spider_output_async(self, response, result, spider): async def process_spider_output_async(self, response, result):
yield yield
@ -600,3 +600,34 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
async def test_exc_async_simple(self): async def test_exc_async_simple(self):
"""Async exc mw -> simple output mw; cannot work as downgrading is not supported""" """Async exc mw -> simple output mw; cannot work as downgrading is not supported"""
await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN) await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN)
class TestDeprecatedSpiderArg(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_deprecated_spider_arg(self):
class DeprecatedSpiderArgMiddleware:
def process_spider_input(self, response, spider):
return None
def process_spider_output(self, response, result, spider):
1 / 0
def process_spider_exception(self, response, exception, spider):
return []
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"process_spider_input\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"process_spider_output\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"process_spider_exception\(\) requires a spider argument",
),
):
self.mwman._add_middleware(DeprecatedSpiderArgMiddleware())
await self._scrape_response()

14
tests/test_spidermiddleware_depth.py
View File

@ -21,12 +21,6 @@ def crawler() -> Crawler:
return get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True}) return get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True})
@pytest.fixture
def spider(crawler: Crawler) -> Spider:
crawler.spider = crawler._create_spider("scrapytest.org")
return crawler.spider
@pytest.fixture @pytest.fixture
def stats(crawler: Crawler) -> Generator[StatsCollector]: def stats(crawler: Crawler) -> Generator[StatsCollector]:
assert crawler.stats is not None assert crawler.stats is not None
@ -42,15 +36,13 @@ def mw(crawler: Crawler) -> DepthMiddleware:
return DepthMiddleware.from_crawler(crawler) return DepthMiddleware.from_crawler(crawler)
def test_process_spider_output( def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> None:
mw: DepthMiddleware, stats: StatsCollector, spider: Spider
) -> None:
req = Request("http://scrapytest.org") req = Request("http://scrapytest.org")
resp = Response("http://scrapytest.org") resp = Response("http://scrapytest.org")
resp.request = req resp.request = req
result = [Request("http://scrapytest.org")] result = [Request("http://scrapytest.org")]
out = list(mw.process_spider_output(resp, result, spider)) out = list(mw.process_spider_output(resp, result))
assert out == result assert out == result
rdc = stats.get_value("request_depth_count/1") rdc = stats.get_value("request_depth_count/1")
@ -58,7 +50,7 @@ def test_process_spider_output(
req.meta["depth"] = 1 req.meta["depth"] = 1
out2 = list(mw.process_spider_output(resp, result, spider)) out2 = list(mw.process_spider_output(resp, result))
assert not out2 assert not out2
rdm = stats.get_value("request_depth_max") rdm = stats.get_value("request_depth_max")

95
tests/test_spidermiddleware_httperror.py
View File

@ -7,9 +7,8 @@ from testfixtures import LogCapture
from twisted.internet.defer import inlineCallbacks from twisted.internet.defer import inlineCallbacks
from scrapy.http import Request, Response from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware
from scrapy.spiders import Spider from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
from tests.spiders import MockServerSpider from tests.spiders import MockServerSpider
@ -57,12 +56,6 @@ def _response(request: Request, status_code: int) -> Response:
return Response(request.url, status=status_code, request=request) return Response(request.url, status=status_code, request=request)
@pytest.fixture
def spider() -> Spider:
crawler = get_crawler(Spider)
return Spider.from_crawler(crawler, name="foo")
@pytest.fixture @pytest.fixture
def res200() -> Response: def res200() -> Response:
return _response(req, 200) return _response(req, 200)
@ -81,36 +74,35 @@ def res404() -> Response:
class TestHttpErrorMiddleware: class TestHttpErrorMiddleware:
@pytest.fixture @pytest.fixture
def mw(self) -> HttpErrorMiddleware: def mw(self) -> HttpErrorMiddleware:
return HttpErrorMiddleware(Settings({})) crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
return HttpErrorMiddleware.from_crawler(crawler)
def test_process_spider_input( def test_process_spider_input(
self, self, mw: HttpErrorMiddleware, res200: Response, res404: Response
mw: HttpErrorMiddleware,
spider: Spider,
res200: Response,
res404: Response,
) -> None: ) -> None:
mw.process_spider_input(res200, spider) mw.process_spider_input(res200)
with pytest.raises(HttpError): with pytest.raises(HttpError):
mw.process_spider_input(res404, spider) mw.process_spider_input(res404)
def test_process_spider_exception( def test_process_spider_exception(
self, mw: HttpErrorMiddleware, spider: Spider, res404: Response self, mw: HttpErrorMiddleware, res404: Response
) -> None: ) -> None:
assert mw.process_spider_exception(res404, HttpError(res404), spider) == [] assert mw.process_spider_exception(res404, HttpError(res404)) == []
assert mw.process_spider_exception(res404, Exception(), spider) is None assert mw.process_spider_exception(res404, Exception()) is None
def test_handle_httpstatus_list( def test_handle_httpstatus_list(
self, mw: HttpErrorMiddleware, spider: Spider, res404: Response self, mw: HttpErrorMiddleware, res404: Response
) -> None: ) -> None:
request = Request( request = Request(
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]} "http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
) )
res = _response(request, 404) res = _response(request, 404)
mw.process_spider_input(res, spider) mw.process_spider_input(res)
spider.handle_httpstatus_list = [404] # type: ignore[attr-defined] assert mw.crawler.spider
mw.process_spider_input(res404, spider) mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
mw.process_spider_input(res404)
class TestHttpErrorMiddlewareSettings: class TestHttpErrorMiddlewareSettings:
@ -118,76 +110,71 @@ class TestHttpErrorMiddlewareSettings:
@pytest.fixture @pytest.fixture
def mw(self) -> HttpErrorMiddleware: def mw(self) -> HttpErrorMiddleware:
return HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)})) crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOWED_CODES": (402,)})
crawler.spider = crawler._create_spider()
return HttpErrorMiddleware.from_crawler(crawler)
def test_process_spider_input( def test_process_spider_input(
self, self,
mw: HttpErrorMiddleware, mw: HttpErrorMiddleware,
spider: Spider,
res200: Response, res200: Response,
res402: Response, res402: Response,
res404: Response, res404: Response,
) -> None: ) -> None:
mw.process_spider_input(res200, spider) mw.process_spider_input(res200)
with pytest.raises(HttpError): with pytest.raises(HttpError):
mw.process_spider_input(res404, spider) mw.process_spider_input(res404)
mw.process_spider_input(res402, spider) mw.process_spider_input(res402)
def test_meta_overrides_settings( def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None:
self, mw: HttpErrorMiddleware, spider: Spider
) -> None:
request = Request( request = Request(
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]} "http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
) )
res404 = _response(request, 404) res404 = _response(request, 404)
res402 = _response(request, 402) res402 = _response(request, 402)
mw.process_spider_input(res404, spider) mw.process_spider_input(res404)
with pytest.raises(HttpError): with pytest.raises(HttpError):
mw.process_spider_input(res402, spider) mw.process_spider_input(res402)
def test_spider_override_settings( def test_spider_override_settings(
self, self, mw: HttpErrorMiddleware, res402: Response, res404: Response
mw: HttpErrorMiddleware,
spider: Spider,
res402: Response,
res404: Response,
) -> None: ) -> None:
spider.handle_httpstatus_list = [404] # type: ignore[attr-defined] assert mw.crawler.spider
mw.process_spider_input(res404, spider) mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
mw.process_spider_input(res404)
with pytest.raises(HttpError): with pytest.raises(HttpError):
mw.process_spider_input(res402, spider) mw.process_spider_input(res402)
class TestHttpErrorMiddlewareHandleAll: class TestHttpErrorMiddlewareHandleAll:
@pytest.fixture @pytest.fixture
def mw(self) -> HttpErrorMiddleware: def mw(self) -> HttpErrorMiddleware:
return HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True})) crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOW_ALL": True})
crawler.spider = crawler._create_spider()
return HttpErrorMiddleware.from_crawler(crawler)
def test_process_spider_input( def test_process_spider_input(
self, self,
mw: HttpErrorMiddleware, mw: HttpErrorMiddleware,
spider: Spider,
res200: Response, res200: Response,
res404: Response, res404: Response,
) -> None: ) -> None:
mw.process_spider_input(res200, spider) mw.process_spider_input(res200)
mw.process_spider_input(res404, spider) mw.process_spider_input(res404)
def test_meta_overrides_settings( def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None:
self, mw: HttpErrorMiddleware, spider: Spider
) -> None:
request = Request( request = Request(
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]} "http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
) )
res404 = _response(request, 404) res404 = _response(request, 404)
res402 = _response(request, 402) res402 = _response(request, 402)
mw.process_spider_input(res404, spider) mw.process_spider_input(res404)
with pytest.raises(HttpError): with pytest.raises(HttpError):
mw.process_spider_input(res402, spider) mw.process_spider_input(res402)
def test_httperror_allow_all_false(self, spider: Spider) -> None: def test_httperror_allow_all_false(self) -> None:
crawler = get_crawler(_HttpErrorSpider) crawler = get_crawler(_HttpErrorSpider)
mw = HttpErrorMiddleware.from_crawler(crawler) mw = HttpErrorMiddleware.from_crawler(crawler)
request_httpstatus_false = Request( request_httpstatus_false = Request(
@ -200,8 +187,8 @@ class TestHttpErrorMiddlewareHandleAll:
res402 = _response(request_httpstatus_true, 402) res402 = _response(request_httpstatus_true, 402)
with pytest.raises(HttpError): with pytest.raises(HttpError):
mw.process_spider_input(res404, spider) mw.process_spider_input(res404)
mw.process_spider_input(res402, spider) mw.process_spider_input(res402)
class TestHttpErrorMiddlewareIntegrational: class TestHttpErrorMiddlewareIntegrational:
@ -218,7 +205,7 @@ class TestHttpErrorMiddlewareIntegrational:
def test_middleware_works(self): def test_middleware_works(self):
crawler = get_crawler(_HttpErrorSpider) crawler = get_crawler(_HttpErrorSpider)
yield crawler.crawl(mockserver=self.mockserver) yield crawler.crawl(mockserver=self.mockserver)
assert not crawler.spider.skipped, crawler.spider.skipped assert not crawler.spider.skipped
assert crawler.spider.parsed == {"200"} assert crawler.spider.parsed == {"200"}
assert crawler.spider.failed == {"404", "402", "500"} assert crawler.spider.failed == {"404", "402", "500"}

87
tests/test_spidermiddleware_output_chain.py
View File

@ -6,18 +6,27 @@ from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer from tests.mockserver.http import MockServer
class LogExceptionMiddleware: class _BaseSpiderMiddleware:
def process_spider_exception(self, response, exception, spider): def __init__(self, crawler):
spider.logger.info( self.crawler = crawler
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
class LogExceptionMiddleware(_BaseSpiderMiddleware):
def process_spider_exception(self, response, exception):
self.crawler.spider.logger.info(
"Middleware: %s exception caught", exception.__class__.__name__ "Middleware: %s exception caught", exception.__class__.__name__
) )
# ================================================================================ # ================================================================================
# (0) recover from an exception on a spider callback # (0) recover from an exception on a spider callback
class RecoveryMiddleware: class RecoveryMiddleware(_BaseSpiderMiddleware):
def process_spider_exception(self, response, exception, spider): def process_spider_exception(self, response, exception):
spider.logger.info( self.crawler.spider.logger.info(
"Middleware: %s exception caught", exception.__class__.__name__ "Middleware: %s exception caught", exception.__class__.__name__
) )
return [ return [
@ -55,9 +64,9 @@ class RecoveryAsyncGenSpider(RecoverySpider):
# ================================================================================ # ================================================================================
# (1) exceptions from a spider middleware's process_spider_input method # (1) exceptions from a spider middleware's process_spider_input method
class FailProcessSpiderInputMiddleware: class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware):
def process_spider_input(self, response, spider): def process_spider_input(self, response):
spider.logger.info("Middleware: will raise IndexError") self.crawler.spider.logger.info("Middleware: will raise IndexError")
raise IndexError raise IndexError
@ -159,27 +168,31 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS
# ================================================================================ # ================================================================================
# (4) exceptions from a middleware process_spider_output method (generator) # (4) exceptions from a middleware process_spider_output method (generator)
class _GeneratorDoNothingMiddleware: class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware):
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
for r in result: for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output") r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r yield r
def process_spider_exception(self, response, exception, spider): def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception" method = f"{self.__class__.__name__}.process_spider_exception"
spider.logger.info("%s: %s caught", method, exception.__class__.__name__) self.crawler.spider.logger.info(
"%s: %s caught", method, exception.__class__.__name__
)
class GeneratorFailMiddleware: class GeneratorFailMiddleware(_BaseSpiderMiddleware):
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
for r in result: for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output") r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r yield r
raise LookupError raise LookupError
def process_spider_exception(self, response, exception, spider): def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception" method = f"{self.__class__.__name__}.process_spider_exception"
spider.logger.info("%s: %s caught", method, exception.__class__.__name__) self.crawler.spider.logger.info(
"%s: %s caught", method, exception.__class__.__name__
)
yield {"processed": [method]} yield {"processed": [method]}
@ -187,15 +200,17 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware):
pass pass
class GeneratorRecoverMiddleware: class GeneratorRecoverMiddleware(_BaseSpiderMiddleware):
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
for r in result: for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output") r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r yield r
def process_spider_exception(self, response, exception, spider): def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception" method = f"{self.__class__.__name__}.process_spider_exception"
spider.logger.info("%s: %s caught", method, exception.__class__.__name__) self.crawler.spider.logger.info(
"%s: %s caught", method, exception.__class__.__name__
)
yield {"processed": [method]} yield {"processed": [method]}
@ -226,30 +241,34 @@ class GeneratorOutputChainSpider(Spider):
# (5) exceptions from a middleware process_spider_output method (not generator) # (5) exceptions from a middleware process_spider_output method (not generator)
class _NotGeneratorDoNothingMiddleware: class _NotGeneratorDoNothingMiddleware(_BaseSpiderMiddleware):
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
out = [] out = []
for r in result: for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output") r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r) out.append(r)
return out return out
def process_spider_exception(self, response, exception, spider): def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception" method = f"{self.__class__.__name__}.process_spider_exception"
spider.logger.info("%s: %s caught", method, exception.__class__.__name__) self.crawler.spider.logger.info(
"%s: %s caught", method, exception.__class__.__name__
)
class NotGeneratorFailMiddleware: class NotGeneratorFailMiddleware(_BaseSpiderMiddleware):
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
out = [] out = []
for r in result: for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output") r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r) out.append(r)
raise ReferenceError raise ReferenceError
def process_spider_exception(self, response, exception, spider): def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception" method = f"{self.__class__.__name__}.process_spider_exception"
spider.logger.info("%s: %s caught", method, exception.__class__.__name__) self.crawler.spider.logger.info(
"%s: %s caught", method, exception.__class__.__name__
)
return [{"processed": [method]}] return [{"processed": [method]}]
@ -257,17 +276,19 @@ class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddlewa
pass pass
class NotGeneratorRecoverMiddleware: class NotGeneratorRecoverMiddleware(_BaseSpiderMiddleware):
def process_spider_output(self, response, result, spider): def process_spider_output(self, response, result):
out = [] out = []
for r in result: for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output") r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r) out.append(r)
return out return out
def process_spider_exception(self, response, exception, spider): def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception" method = f"{self.__class__.__name__}.process_spider_exception"
spider.logger.info("%s: %s caught", method, exception.__class__.__name__) self.crawler.spider.logger.info(
"%s: %s caught", method, exception.__class__.__name__
)
return [{"processed": [method]}] return [{"processed": [method]}]

6
tests/test_spidermiddleware_referer.py
View File

@ -61,11 +61,11 @@ class TestRefererMiddleware:
def get_response(self, origin: str) -> Response: def get_response(self, origin: str) -> Response:
return Response(origin, headers=self.resp_headers) return Response(origin, headers=self.resp_headers)
def test(self, mw: RefererMiddleware, spider: Spider) -> None: def test(self, mw: RefererMiddleware) -> None:
for origin, target, referrer in self.scenarii: for origin, target, referrer in self.scenarii:
response = self.get_response(origin) response = self.get_response(origin)
request = self.get_request(target) request = self.get_request(target)
out = list(mw.process_spider_output(response, [request], spider)) out = list(mw.process_spider_output(response, [request]))
assert out[0].headers.get("Referer") == referrer assert out[0].headers.get("Referer") == referrer
@ -1036,7 +1036,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
response = self.get_response(parent) response = self.get_response(parent)
request = self.get_request(target) request = self.get_request(target)
out = list(referrermw.process_spider_output(response, [request], spider)) out = list(referrermw.process_spider_output(response, [request]))
assert out[0].headers.get("Referer") == init_referrer assert out[0].headers.get("Referer") == init_referrer
for status, url in redirections: for status, url in redirections:

20
tests/test_spidermiddleware_urllength.py
View File

@ -27,11 +27,6 @@ def crawler() -> Crawler:
return get_crawler(Spider, {"URLLENGTH_LIMIT": maxlength}) return get_crawler(Spider, {"URLLENGTH_LIMIT": maxlength})
@pytest.fixture
def spider(crawler: Crawler) -> Spider:
return crawler._create_spider("foo")
@pytest.fixture @pytest.fixture
def stats(crawler: Crawler) -> StatsCollector: def stats(crawler: Crawler) -> StatsCollector:
assert crawler.stats is not None assert crawler.stats is not None
@ -43,22 +38,19 @@ def mw(crawler: Crawler) -> UrlLengthMiddleware:
return UrlLengthMiddleware.from_crawler(crawler) return UrlLengthMiddleware.from_crawler(crawler)
def process_spider_output(mw: UrlLengthMiddleware, spider: Spider) -> list[Request]: def process_spider_output(mw: UrlLengthMiddleware) -> list[Request]:
return list(mw.process_spider_output(response, reqs, spider)) return list(mw.process_spider_output(response, reqs))
def test_middleware_works(mw: UrlLengthMiddleware, spider: Spider) -> None: def test_middleware_works(mw: UrlLengthMiddleware) -> None:
assert process_spider_output(mw, spider) == [short_url_req] assert process_spider_output(mw) == [short_url_req]
def test_logging( def test_logging(
stats: StatsCollector, stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture
mw: UrlLengthMiddleware,
spider: Spider,
caplog: pytest.LogCaptureFixture,
) -> None: ) -> None:
with caplog.at_level(INFO): with caplog.at_level(INFO):
process_spider_output(mw, spider) process_spider_output(mw)
ric = stats.get_value("urllength/request_ignored_count") ric = stats.get_value("urllength/request_ignored_count")
assert ric == 1 assert ric == 1
assert f"Ignoring link (url length > {maxlength})" in caplog.text assert f"Ignoring link (url length > {maxlength})" in caplog.text

22
tests/test_utils_defer.py
View File

@ -10,7 +10,6 @@ from twisted.internet.defer import Deferred, inlineCallbacks, succeed
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import ( from scrapy.utils.defer import (
_process_chain,
aiter_errback, aiter_errback,
deferred_f_from_coro_f, deferred_f_from_coro_f,
deferred_from_coro, deferred_from_coro,
@ -19,7 +18,6 @@ from scrapy.utils.defer import (
maybe_deferred_to_future, maybe_deferred_to_future,
mustbe_deferred, mustbe_deferred,
parallel_async, parallel_async,
process_parallel,
) )
if TYPE_CHECKING: if TYPE_CHECKING:
@ -85,26 +83,6 @@ def eb1(failure, arg1, arg2):
return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})" return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})"
class TestDeferUtils:
@deferred_f_from_coro_f
async def test_process_chain(self):
x = await _process_chain([cb1, cb2, cb3], "res", "v1", "v2")
assert x == "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)"
with pytest.raises(TypeError):
await _process_chain([cb1, cb_fail, cb3], "res", "v1", "v2")
@inlineCallbacks
def test_process_parallel(self):
x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2")
assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"]
@inlineCallbacks
def test_process_parallel_failure(self):
with pytest.raises(TypeError):
yield process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2")
class TestIterErrback: class TestIterErrback:
def test_iter_errback_good(self): def test_iter_errback_good(self):
def itergood() -> Generator[int, None, None]: def itergood() -> Generator[int, None, None]: