RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Deprecate spider args of middlewares and pipelines. (#7006) 

* Deprecate spider args of spider middleware methods.

* Deprecate the spider arg of pipeline process_item().

* Fix a typing issue.

* Deprecate the spider arg of pipeline {open,close}_spider().

* Cleanup.

* Update docs.

* Add pragma: no cover to some deprecated code.

* More tests.

* More tests.

* More tests.

* Simplify _process_parallel().
This commit is contained in:
Andrey Rakhmatullin 2025-08-21 13:23:10 +05:00 committed by GitHub
parent b68330811b
commit ed63fa94d6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
34 changed files with 617 additions and 383 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/faq.rst
View File

@ -349,7 +349,7 @@ method for this purpose. For example:
class MultiplyItemsMiddleware:
def process_spider_output(self, response, result, spider):
def process_spider_output(self, response, result):
for item_or_request in result:
if isinstance(item_or_request, Request):
continue

14
docs/news.rst
View File

@ -44,6 +44,20 @@ Backward-incompatible changes
- :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer`
- :class:`scrapy.middleware.MiddlewareManager` no longer includes code for
handling ``open_spider()`` and ``close_spider()`` component methods. As
this code was only used for pipelines it was moved into
:class:`scrapy.pipelines.ItemPipelineManager`. This change should only
affect custom subclasses of :class:`~scrapy.middleware.MiddlewareManager`.
The following code was moved:
- ``scrapy.middleware.MiddlewareManager.open_spider()``
- ``scrapy.middleware.MiddlewareManager.close_spider()``
- Code in ``scrapy.middleware.MiddlewareManager._add_middleware()`` that
processes ``open_spider()`` and ``close_spider()`` component methods.
- :meth:`scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware.process_request`
now returns a coroutine, previously it returned a
:class:`~twisted.internet.defer.Deferred` object or ``None``. The

8
docs/topics/coroutines.rst
View File

@ -191,7 +191,7 @@ shorter and cleaner:
adapter["field"] = data
return item
def process_item(self, item, spider):
def process_item(self, item):
adapter = ItemAdapter(item)
dfd = db.get_some_data(adapter["id"])
dfd.addCallback(self._update_item, item)
@ -205,7 +205,7 @@ becomes:
class DbPipeline:
async def process_item(self, item, spider):
async def process_item(self, item):
adapter = ItemAdapter(item)
adapter["field"] = await db.get_some_data(adapter["id"])
return item
@ -421,12 +421,12 @@ For example:
.. code-block:: python
class UniversalSpiderMiddleware:
def process_spider_output(self, response, result, spider):
def process_spider_output(self, response, result):
for r in result:
# ... do something with r
yield r
async def process_spider_output_async(self, response, result, spider):
async def process_spider_output_async(self, response, result):
async for r in result:
# ... do something with r
yield r

2
docs/topics/exporters.rst
View File

@ -67,7 +67,7 @@ value of one of their fields:
self.year_to_exporter[year] = (exporter, xml_file)
return self.year_to_exporter[year][0]
def process_item(self, item, spider):
def process_item(self, item):
exporter = self._exporter_for_item(item)
exporter.export_item(item)
return item

42
docs/topics/item-pipeline.rst
View File

@ -26,7 +26,7 @@ Writing your own item pipeline
Each item pipeline is a :ref:`component <topics-components>` that must
implement the following method:
.. method:: process_item(self, item, spider)
.. method:: process_item(self, item)
This method is called for every item pipeline component.
@ -42,25 +42,16 @@ implement the following method:
:param item: the scraped item
:type item: :ref:`item object <item-types>`
:param spider: the spider which scraped the item
:type spider: :class:`~scrapy.Spider` object
Additionally, they may also implement the following methods:
.. method:: open_spider(self, spider)
.. method:: open_spider(self)
This method is called when the spider is opened.
:param spider: the spider which was opened
:type spider: :class:`~scrapy.Spider` object
.. method:: close_spider(self, spider)
.. method:: close_spider(self)
This method is called when the spider is closed.
:param spider: the spider which was closed
:type spider: :class:`~scrapy.Spider` object
Item pipeline example
=====================
@ -82,7 +73,7 @@ contain a price:
class PricePipeline:
vat_factor = 1.15
def process_item(self, item, spider):
def process_item(self, item):
adapter = ItemAdapter(item)
if adapter.get("price"):
if adapter.get("price_excludes_vat"):
@ -107,13 +98,13 @@ format:
class JsonWriterPipeline:
def open_spider(self, spider):
def open_spider(self):
self.file = open("items.jsonl", "w")
def close_spider(self, spider):
def close_spider(self):
self.file.close()
def process_item(self, item, spider):
def process_item(self, item):
line = json.dumps(ItemAdapter(item).asdict()) + "\n"
self.file.write(line)
return item
@ -153,14 +144,14 @@ The main point of this example is to show how to :ref:`get the crawler
mongo_db=crawler.settings.get("MONGO_DATABASE", "items"),
)
def open_spider(self, spider):
def open_spider(self):
self.client = pymongo.MongoClient(self.mongo_uri)
self.db = self.client[self.mongo_db]
def close_spider(self, spider):
def close_spider(self):
self.client.close()
def process_item(self, item, spider):
def process_item(self, item):
self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
return item
@ -198,12 +189,19 @@ item.
SPLASH_URL = "http://localhost:8050/render.png?url={}"
async def process_item(self, item, spider):
def __init__(crawler):
self.crawler = crawler
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
async def process_item(self, item):
adapter = ItemAdapter(item)
encoded_item_url = quote(adapter["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url, callback=NO_CALLBACK)
response = await spider.crawler.engine.download_async(request)
response = await self.crawler.engine.download_async(request)
if response.status != 200:
# Error happened, return item.
@ -238,7 +236,7 @@ returns multiples items with the same id:
def __init__(self):
self.ids_seen = set()
def process_item(self, item, spider):
def process_item(self, item):
adapter = ItemAdapter(item)
if adapter["id"] in self.ids_seen:
raise DropItem(f"Item ID already seen: {adapter['id']}")

2
docs/topics/settings.rst
View File

@ -554,7 +554,7 @@ When writing an item pipeline, you can force a different log level by setting
class MyPipeline:
def process_item(self, item, spider):
def process_item(self, item):
if not item.get("price"):
raise DropItem("Missing price data", log_level="INFO")
return item

17
docs/topics/spider-middleware.rst
View File

@ -94,7 +94,7 @@ one or more of these methods:
def process_start_requests(self, start, spider):
yield from start
.. method:: process_spider_input(response, spider)
.. method:: process_spider_input(response)
This method is called for each response that goes through the spider
middleware and into the spider, for processing.
@ -116,11 +116,7 @@ one or more of these methods:
:param response: the response being processed
:type response: :class:`~scrapy.http.Response` object
:param spider: the spider for which this response is intended
:type spider: :class:`~scrapy.Spider` object
.. method:: process_spider_output(response, result, spider)
.. method:: process_spider_output(response, result)
This method is called with the results returned from the Spider, after
it has processed the response.
@ -149,10 +145,7 @@ one or more of these methods:
:type result: an iterable of :class:`~scrapy.Request` objects and
:ref:`item objects <topics-items>`
:param spider: the spider whose result is being processed
:type spider: :class:`~scrapy.Spider` object
.. method:: process_spider_output_async(response, result, spider)
.. method:: process_spider_output_async(response, result)
:async:
.. versionadded:: 2.7
@ -161,7 +154,7 @@ one or more of these methods:
which will be called instead of :meth:`process_spider_output` if
``result`` is an :term:`asynchronous iterable`.
.. method:: process_spider_exception(response, exception, spider)
.. method:: process_spider_exception(response, exception)
This method is called when a spider or :meth:`process_spider_output`
method (from a previous spider middleware) raises an exception.
@ -186,8 +179,6 @@ one or more of these methods:
:param exception: the exception raised
:type exception: :exc:`Exception` object
:param spider: the spider which raised the exception
:type spider: :class:`~scrapy.Spider` object
Base class for custom spider middlewares
----------------------------------------

33
scrapy/core/spidermw.py
View File

@ -117,9 +117,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
)
def _add_middleware(self, mw: Any) -> None:
super()._add_middleware(mw)
if hasattr(mw, "process_spider_input"):
self.methods["process_spider_input"].append(mw.process_spider_input)
self._check_mw_method_spider_arg(mw.process_spider_input)
if self._use_start_requests:
if hasattr(mw, "process_start_requests"):
self.methods["process_start_requests"].appendleft(
@ -127,10 +128,19 @@ class SpiderMiddlewareManager(MiddlewareManager):
)
elif hasattr(mw, "process_start"):
self.methods["process_start"].appendleft(mw.process_start)
process_spider_output = self._get_async_method_pair(mw, "process_spider_output")
self.methods["process_spider_output"].appendleft(process_spider_output)
if callable(process_spider_output):
self._check_mw_method_spider_arg(process_spider_output)
elif isinstance(process_spider_output, tuple):
for m in process_spider_output:
self._check_mw_method_spider_arg(m)
process_spider_exception = getattr(mw, "process_spider_exception", None)
self.methods["process_spider_exception"].appendleft(process_spider_exception)
if process_spider_exception is not None:
self._check_mw_method_spider_arg(process_spider_exception)
async def _process_spider_input(
self,
@ -141,7 +151,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
for method in self.methods["process_spider_input"]:
method = cast("Callable", method)
try:
result = method(response=response, spider=self._spider)
if method in self._mw_methods_requiring_spider:
result = method(response=response, spider=self._spider)
else:
result = method(response=response)
if result is not None:
msg = (
f"{global_object_name(method)} must return None "
@ -212,7 +225,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
if method is None:
continue
method = cast("Callable", method)
result = method(response=response, exception=exception, spider=self._spider)
if method in self._mw_methods_requiring_spider:
result = method(
response=response, exception=exception, spider=self._spider
)
else:
result = method(response=response, exception=exception)
if _isiterable(result):
# stop exception handling by handing control over to the
# process_spider_output chain if an iterable has been returned
@ -298,7 +316,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
)
recovered = MutableChain(recovered_collected)
# might fail directly if the output value is not a generator
result = method(response=response, result=result, spider=self._spider)
if method in self._mw_methods_requiring_spider:
result = method(
response=response, result=result, spider=self._spider
)
else:
result = method(response=response, result=result)
except Exception as ex:
exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = (
self._process_spider_exception(response, ex, method_index + 1)
@ -421,7 +444,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
if self._use_start_requests:
sync_start = iter(self._spider.start_requests())
sync_start = await self._process_chain(
"process_start_requests", sync_start, self._spider
"process_start_requests", sync_start, always_add_spider=True
)
start: AsyncIterator[Any] = as_async_generator(sync_start)
else:

72
scrapy/middleware.py
View File

@ -8,7 +8,8 @@ from collections import defaultdict, deque
from typing import TYPE_CHECKING, Any, TypeVar, cast
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.utils.defer import _process_chain, process_parallel
from scrapy.utils.defer import ensure_awaitable
from scrapy.utils.deprecate import argument_is_required
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.python import global_object_name
@ -31,7 +32,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
_T2 = TypeVar("_T2")
class MiddlewareManager(ABC):
@ -56,6 +56,7 @@ class MiddlewareManager(ABC):
self.methods: dict[str, deque[Callable | tuple[Callable, Callable] | None]] = (
defaultdict(deque)
)
self._mw_methods_requiring_spider: set[Callable] = set()
for mw in middlewares:
self._add_middleware(mw)
@ -158,34 +159,53 @@ class MiddlewareManager(ABC):
)
return cls(*middlewares, crawler=crawler)
def _add_middleware(self, mw: Any) -> None:
if hasattr(mw, "open_spider"):
self.methods["open_spider"].append(mw.open_spider)
if hasattr(mw, "close_spider"):
self.methods["close_spider"].appendleft(mw.close_spider)
def _add_middleware(self, mw: Any) -> None: # noqa: B027
pass
def _process_parallel(
self, methodname: str, obj: _T, *args: Any
) -> Deferred[list[_T2]]:
methods = cast(
"Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname]
)
return process_parallel(methods, obj, *args)
def _check_mw_method_spider_arg(self, method: Callable) -> None:
if argument_is_required(method, "spider"):
warnings.warn(
f"{method.__qualname__}() requires a spider argument,"
f" this is deprecated and the argument will not be passed in future Scrapy versions."
f" If you need to access the spider instance you can save the crawler instance"
f" passed to from_crawler() and use its spider attribute.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
self._mw_methods_requiring_spider.add(method)
async def _process_chain(self, methodname: str, obj: _T, *args: Any) -> _T:
async def _process_chain(
self,
methodname: str,
obj: _T,
*args: Any,
add_spider: bool = False,
always_add_spider: bool = False,
) -> _T:
methods = cast(
"Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname]
)
return await _process_chain(methods, obj, *args)
for method in methods:
if always_add_spider or (
add_spider and method in self._mw_methods_requiring_spider
):
obj = await ensure_awaitable(method(obj, *(*args, self._spider)))
else:
obj = await ensure_awaitable(method(obj, *args))
return obj
def open_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:
if spider:
self._warn_spider_arg("open_spider")
self._set_compat_spider(spider)
return self._process_parallel("open_spider", self._spider)
def open_spider(
self, spider: Spider | None = None
) -> Deferred[list[None]]: # pragma: no cover
raise NotImplementedError(
"MiddlewareManager.open_spider() is no longer implemented"
" and will be removed in a future Scrapy version."
)
def close_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:
if spider:
self._warn_spider_arg("close_spider")
self._set_compat_spider(spider)
return self._process_parallel("close_spider", self._spider)
def close_spider(
self, spider: Spider | None = None
) -> Deferred[list[None]]: # pragma: no cover
raise NotImplementedError(
"MiddlewareManager.close_spider() is no longer implemented"
" and will be removed in a future Scrapy version."
)

52
scrapy/pipelines/__init__.py
View File

@ -7,16 +7,20 @@ See documentation in docs/item-pipeline.rst
from __future__ import annotations
import warnings
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, Callable, cast
from twisted.internet.defer import Deferred, DeferredList
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.middleware import MiddlewareManager
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.defer import deferred_from_coro, maybeDeferred_coro
from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
from collections.abc import Iterable
from twisted.python.failure import Failure
from scrapy import Spider
from scrapy.settings import Settings
@ -30,9 +34,15 @@ class ItemPipelineManager(MiddlewareManager):
return build_component_list(settings.getwithbase("ITEM_PIPELINES"))
def _add_middleware(self, pipe: Any) -> None:
super()._add_middleware(pipe)
if hasattr(pipe, "open_spider"):
self.methods["open_spider"].append(pipe.open_spider)
self._check_mw_method_spider_arg(pipe.open_spider)
if hasattr(pipe, "close_spider"):
self.methods["close_spider"].appendleft(pipe.close_spider)
self._check_mw_method_spider_arg(pipe.close_spider)
if hasattr(pipe, "process_item"):
self.methods["process_item"].append(pipe.process_item)
self._check_mw_method_spider_arg(pipe.process_item)
def process_item(self, item: Any, spider: Spider | None = None) -> Deferred[Any]:
if spider:
@ -45,4 +55,36 @@ class ItemPipelineManager(MiddlewareManager):
return deferred_from_coro(self.process_item_async(item))
async def process_item_async(self, item: Any) -> Any:
return await self._process_chain("process_item", item, self._spider)
return await self._process_chain("process_item", item, add_spider=True)
def _process_parallel(self, methodname: str) -> Deferred[list[None]]:
methods = cast("Iterable[Callable[..., None]]", self.methods[methodname])
def get_dfd(method: Callable[..., None]) -> Deferred[None]:
if method in self._mw_methods_requiring_spider:
return maybeDeferred_coro(method, self._spider)
return maybeDeferred_coro(method)
dfds = [get_dfd(m) for m in methods]
d: Deferred[list[tuple[bool, None]]] = DeferredList(
dfds, fireOnOneErrback=True, consumeErrors=True
)
d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r])
def eb(failure: Failure) -> Failure:
return failure.value.subFailure
d2.addErrback(eb)
return d2
def open_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:
if spider:
self._warn_spider_arg("open_spider")
self._set_compat_spider(spider)
return self._process_parallel("open_spider")
def close_spider(self, spider: Spider | None = None) -> Deferred[list[None]]:
if spider:
self._warn_spider_arg("close_spider")
self._set_compat_spider(spider)
return self._process_parallel("close_spider")

21
scrapy/pipelines/media.py
View File

@ -165,12 +165,27 @@ class MediaPipeline(ABC):
pipe._finish_init(crawler)
return pipe
def open_spider(self, spider: Spider) -> None:
self.spiderinfo = self.SpiderInfo(spider)
def open_spider(self, spider: Spider | None = None) -> None:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to MediaPipeline.open_spider()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
assert self.crawler.spider
self.spiderinfo = self.SpiderInfo(self.crawler.spider)
def process_item(
self, item: Any, spider: Spider
self, item: Any, spider: Spider | None = None
) -> Deferred[list[FileInfoOrError]]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to MediaPipeline.process_item()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
info = self.spiderinfo
requests = arg_to_iter(self.get_media_requests(item, info))
dlist = [self._process_request(r, info, item) for r in requests]

23
scrapy/spidermiddlewares/base.py
View File

@ -1,8 +1,10 @@
from __future__ import annotations
import warnings
from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider
from scrapy.exceptions import ScrapyDeprecationWarning
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Iterable
@ -53,15 +55,32 @@ class BaseSpiderMiddleware:
yield o
def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider
self, response: Response, result: Iterable[Any], spider: Spider | None = None
) -> Iterable[Any]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to BaseSpiderMiddleware.process_spider_output()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
for o in result:
if (o := self._get_processed(o, response)) is not None:
yield o
async def process_spider_output_async(
self, response: Response, result: AsyncIterator[Any], spider: Spider
self,
response: Response,
result: AsyncIterator[Any],
spider: Spider | None = None,
) -> AsyncIterator[Any]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to BaseSpiderMiddleware.process_spider_output_async()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
async for o in result:
if (o := self._get_processed(o, response)) is not None:
yield o

27
scrapy/spidermiddlewares/depth.py
View File

@ -7,8 +7,10 @@ See documentation in docs/topics/spider-middleware.rst
from __future__ import annotations
import logging
import warnings
from typing import TYPE_CHECKING, Any
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
if TYPE_CHECKING:
@ -53,16 +55,33 @@ class DepthMiddleware(BaseSpiderMiddleware):
return o
def process_spider_output(
self, response: Response, result: Iterable[Any], spider: Spider
self, response: Response, result: Iterable[Any], spider: Spider | None = None
) -> Iterable[Any]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to DepthMiddleware.process_spider_output()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
self._init_depth(response)
yield from super().process_spider_output(response, result, spider)
yield from super().process_spider_output(response, result)
async def process_spider_output_async(
self, response: Response, result: AsyncIterator[Any], spider: Spider
self,
response: Response,
result: AsyncIterator[Any],
spider: Spider | None = None,
) -> AsyncIterator[Any]:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to DepthMiddleware.process_spider_output_async()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
self._init_depth(response)
async for o in super().process_spider_output_async(response, result, spider):
async for o in super().process_spider_output_async(response, result):
yield o
def _init_depth(self, response: Response) -> None:

45
scrapy/spidermiddlewares/httperror.py
View File

@ -7,9 +7,10 @@ See documentation in docs/topics/spider-middleware.rst
from __future__ import annotations
import logging
import warnings
from typing import TYPE_CHECKING, Any
from scrapy.exceptions import IgnoreRequest
from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
if TYPE_CHECKING:
from collections.abc import Iterable
@ -35,9 +36,7 @@ class HttpError(IgnoreRequest):
class HttpErrorMiddleware:
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
return cls(crawler.settings)
crawler: Crawler
def __init__(self, settings: BaseSettings):
self.handle_httpstatus_all: bool = settings.getbool("HTTPERROR_ALLOW_ALL")
@ -45,7 +44,22 @@ class HttpErrorMiddleware:
"HTTPERROR_ALLOWED_CODES"
)
def process_spider_input(self, response: Response, spider: Spider) -> None:
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
o = cls(crawler.settings)
o.crawler = crawler
return o
def process_spider_input(
self, response: Response, spider: Spider | None = None
) -> None:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to HttpErrorMiddleware.process_spider_input()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
if 200 <= response.status < 300: # common case
return
meta = response.meta
@ -57,25 +71,34 @@ class HttpErrorMiddleware:
return
else:
allowed_statuses = getattr(
spider, "handle_httpstatus_list", self.handle_httpstatus_list
self.crawler.spider,
"handle_httpstatus_list",
self.handle_httpstatus_list,
)
if response.status in allowed_statuses:
return
raise HttpError(response, "Ignoring non-200 response")
def process_spider_exception(
self, response: Response, exception: Exception, spider: Spider
self, response: Response, exception: Exception, spider: Spider | None = None
) -> Iterable[Any] | None:
if spider is not None: # pragma: no cover
warnings.warn(
"Passing a spider argument to HttpErrorMiddleware.process_spider_exception()"
" is deprecated and the passed value is ignored.",
ScrapyDeprecationWarning,
stacklevel=2,
)
if isinstance(exception, HttpError):
assert spider.crawler.stats
spider.crawler.stats.inc_value("httperror/response_ignored_count")
spider.crawler.stats.inc_value(
assert self.crawler.stats
self.crawler.stats.inc_value("httperror/response_ignored_count")
self.crawler.stats.inc_value(
f"httperror/response_ignored_status_count/{response.status}"
)
logger.info(
"Ignoring response %(response)r: HTTP status code is not handled or not allowed",
{"response": response},
extra={"spider": spider},
extra={"spider": self.crawler.spider},
)
return []
return None

20
scrapy/utils/defer.py
View File

@ -312,19 +312,6 @@ def process_chain(
return d
async def _process_chain(
callables: Iterable[Callable[Concatenate[_T, _P], _T | Awaitable[_T]]],
input_: _T,
*a: _P.args,
**kw: _P.kwargs,
) -> _T:
"""Chain the given (potentialy asynchronous) callables."""
result = input_
for callable_ in callables:
result = await ensure_awaitable(callable_(result, *a, **kw))
return result
def process_chain_both(
callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]],
@ -355,10 +342,15 @@ def process_parallel(
input: _T, # noqa: A002
*a: _P.args,
**kw: _P.kwargs,
) -> Deferred[list[_T2]]:
) -> Deferred[list[_T2]]: # pragma: no cover
"""Return a Deferred with the output of all successful calls to the given
callbacks
"""
warnings.warn(
"process_parallel() is deprecated.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
dfds = [succeed(input).addCallback(x, *a, **kw) for x in callbacks]
d: Deferred[list[tuple[bool, _T2]]] = DeferredList(
dfds, fireOnOneErrback=True, consumeErrors=True

2
tests/AsyncCrawlerProcess/asyncio_deferred_signal.py
View File

@ -16,7 +16,7 @@ class UppercasePipeline:
def open_spider(self, spider):
return deferred_from_coro(self._open_spider(spider))
def process_item(self, item, spider):
def process_item(self, item):
return {"url": item["url"].upper()}

2
tests/CrawlerProcess/asyncio_deferred_signal.py
View File

@ -16,7 +16,7 @@ class UppercasePipeline:
def open_spider(self, spider):
return deferred_from_coro(self._open_spider(spider))
def process_item(self, item, spider):
def process_item(self, item):
return {"url": item["url"].upper()}

6
tests/pipelines.py
View File

@ -4,13 +4,13 @@ Some pipelines used for testing
class ZeroDivisionErrorPipeline:
def open_spider(self, spider):
def open_spider(self):
1 / 0
def process_item(self, item, spider):
def process_item(self, item):
return item
class ProcessWithZeroDivisionErrorPipeline:
def process_item(self, item, spider):
def process_item(self, item):
1 / 0

4
tests/test_cmdline_crawl_with_pipeline/test_spider/pipelines.py
View File

@ -2,7 +2,7 @@ class TestSpiderPipeline:
def open_spider(self, spider):
pass
def process_item(self, item, spider):
def process_item(self, item):
return item
@ -10,5 +10,5 @@ class TestSpiderExceptionPipeline:
def open_spider(self, spider):
raise RuntimeError("exception")
def process_item(self, item, spider):
def process_item(self, item):
return item

2
tests/test_command_parse.py
View File

@ -153,7 +153,7 @@ import logging
class MyPipeline:
component_name = 'my_pipeline'
def process_item(self, item, spider):
def process_item(self, item):
logging.info('It Works!')
return item
""",

2
tests/test_logformatter.py
View File

@ -246,7 +246,7 @@ class SkipMessagesLogFormatter(LogFormatter):
class DropSomeItemsPipeline:
drop = True
def process_item(self, item, spider):
def process_item(self, item):
if self.drop:
self.drop = False
raise DropItem("Ignoring item")

76
tests/test_middleware.py
View File

@ -7,7 +7,6 @@ import pytest
from scrapy import Spider
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.middleware import MiddlewareManager
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@ -57,7 +56,10 @@ class MyMiddlewareManager(MiddlewareManager):
return [M1, MOff, M3]
def _add_middleware(self, mw):
super()._add_middleware(mw)
if hasattr(mw, "open_spider"):
self.methods["open_spider"].append(mw.open_spider)
if hasattr(mw, "close_spider"):
self.methods["close_spider"].appendleft(mw.close_spider)
if hasattr(mw, "process"):
self.methods["process"].append(mw.process)
@ -105,73 +107,3 @@ def test_no_crawler() -> None:
mwman = MyMiddlewareManager(m1, m2, m3)
assert mwman.middlewares == (m1, m2, m3)
assert mwman.crawler is None
def test_deprecated_spider_arg_no_crawler_spider(crawler: Crawler) -> None:
"""Crawler is provided, but doesn't have a spider. The instance passed to the method is
ignored and raises a warning."""
mwman = MyMiddlewareManager(crawler=crawler)
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated",
),
pytest.raises(
ValueError,
match="MyMiddlewareManager needs to access self.crawler.spider but it is None",
),
):
mwman.open_spider(DefaultSpider())
def test_deprecated_spider_arg_with_crawler(crawler: Crawler) -> None:
"""Crawler is provided and has a spider, works. The instance passed to the method is ignored,
even if mismatched, but raises a warning."""
mwman = MyMiddlewareManager(crawler=crawler)
crawler.spider = crawler._create_spider("foo")
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated",
):
mwman.open_spider(DefaultSpider())
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to MyMiddlewareManager.close_spider\(\) is deprecated",
):
mwman.close_spider(DefaultSpider())
def test_deprecated_spider_arg_without_crawler() -> None:
"""The first instance passed to the method is used, with a warning. Mismatched ones raise an error."""
with pytest.warns(
ScrapyDeprecationWarning,
match="was called without the crawler argument",
):
mwman = MyMiddlewareManager()
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to MyMiddlewareManager.open_spider\(\) is deprecated",
):
mwman.open_spider(DefaultSpider())
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to MyMiddlewareManager.close_spider\(\) is deprecated",
),
pytest.raises(RuntimeError, match="Different instances of Spider were passed"),
):
mwman.close_spider(DefaultSpider())
def test_no_spider_arg_without_crawler() -> None:
"""If no crawler and no spider arg, raise an error."""
with pytest.warns(
ScrapyDeprecationWarning,
match="was called without the crawler argument",
):
mwman = MyMiddlewareManager()
with pytest.raises(
ValueError,
match="has no known Spider instance",
):
mwman.open_spider()

6
tests/test_pipeline_files.py
View File

@ -29,6 +29,7 @@ from scrapy.pipelines.files import (
GCSFilesStore,
S3FilesStore,
)
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
@ -78,10 +79,11 @@ class TestFilesPipeline:
def setup_method(self):
self.tempdir = mkdtemp()
settings_dict = {"FILES_STORE": self.tempdir}
crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
crawler.spider = crawler._create_spider()
self.pipeline = FilesPipeline.from_crawler(crawler)
self.pipeline.download_func = _mocked_download_func
self.pipeline.open_spider(None)
self.pipeline.open_spider()
def teardown_method(self):
rmtree(self.tempdir)

40
tests/test_pipeline_media.py
View File

@ -13,10 +13,10 @@ from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FileException
from scrapy.pipelines.media import MediaPipeline
from scrapy.spiders import Spider
from scrapy.utils.asyncio import call_later
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.signal import disconnect_all
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
@ -48,12 +48,11 @@ class TestBaseMediaPipeline:
settings = None
def setup_method(self):
spider_cls = Spider
self.spider = spider_cls("media.com")
crawler = get_crawler(spider_cls, self.settings)
crawler = get_crawler(DefaultSpider, self.settings)
crawler.spider = crawler._create_spider()
self.pipe = self.pipeline_class.from_crawler(crawler)
self.pipe.download_func = _mocked_download_func
self.pipe.open_spider(self.spider)
self.pipe.open_spider()
self.info = self.pipe.spiderinfo
self.fingerprint = crawler.request_fingerprinter.fingerprint
@ -162,7 +161,7 @@ class TestBaseMediaPipeline:
@inlineCallbacks
def test_default_process_item(self):
item = {"name": "name"}
new_item = yield self.pipe.process_item(item, self.spider)
new_item = yield self.pipe.process_item(item)
assert new_item is item
@ -216,7 +215,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
errback=self._errback,
)
item = {"requests": req}
new_item = yield self.pipe.process_item(item, self.spider)
new_item = yield self.pipe.process_item(item)
assert new_item["results"] == [(True, {})]
assert self.pipe._mockcalled == [
"get_media_requests",
@ -236,7 +235,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
errback=self._errback,
)
item = {"requests": req}
new_item = yield self.pipe.process_item(item, self.spider)
new_item = yield self.pipe.process_item(item)
assert len(new_item["results"]) == 1
assert new_item["results"][0][0] is False
assert isinstance(new_item["results"][0][1], Failure)
@ -258,7 +257,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
fail = Failure(exc)
req2 = Request("http://url2", meta={"response": fail})
item = {"requests": [req1, req2]}
new_item = yield self.pipe.process_item(item, self.spider)
new_item = yield self.pipe.process_item(item)
assert len(new_item["results"]) == 2
assert new_item["results"][0] == (True, {})
assert new_item["results"][1][0] is False
@ -281,7 +280,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
# returns single Request (without callback)
req = Request("http://url")
item = {"requests": req} # pass a single item
new_item = yield self.pipe.process_item(item, self.spider)
new_item = yield self.pipe.process_item(item)
assert new_item is item
assert self.fingerprint(req) in self.info.downloaded
@ -289,7 +288,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
req1 = Request("http://url1")
req2 = Request("http://url2")
item = {"requests": iter([req1, req2])}
new_item = yield self.pipe.process_item(item, self.spider)
new_item = yield self.pipe.process_item(item)
assert new_item is item
assert self.fingerprint(req1) in self.info.downloaded
assert self.fingerprint(req2) in self.info.downloaded
@ -299,7 +298,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
rsp1 = Response("http://url1")
req1 = Request("http://url1", meta={"response": rsp1})
item = {"requests": req1}
new_item = yield self.pipe.process_item(item, self.spider)
new_item = yield self.pipe.process_item(item)
assert new_item is item
assert new_item["results"] == [(True, {})]
@ -308,7 +307,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
req1.url, meta={"response": Response("http://donot.download.me")}
)
item = {"requests": req2}
new_item = yield self.pipe.process_item(item, self.spider)
new_item = yield self.pipe.process_item(item)
assert new_item is item
assert self.fingerprint(req1) == self.fingerprint(req2)
assert new_item["results"] == [(True, {})]
@ -321,7 +320,7 @@ class TestMediaPipeline(TestBaseMediaPipeline):
req1.url, meta={"response": Response("http://donot.download.me")}
)
item = {"requests": [req1, req2]}
new_item = yield self.pipe.process_item(item, self.spider)
new_item = yield self.pipe.process_item(item)
assert new_item is item
assert new_item["results"] == [(True, {}), (True, {})]
@ -348,14 +347,14 @@ class TestMediaPipeline(TestBaseMediaPipeline):
req1 = Request("http://url", meta={"response": rsp1_func})
req2 = Request(req1.url, meta={"response": rsp2_func})
item = {"requests": [req1, req2]}
new_item = yield self.pipe.process_item(item, self.spider)
new_item = yield self.pipe.process_item(item)
assert new_item["results"] == [(True, {}), (True, {})]
@inlineCallbacks
def test_use_media_to_download_result(self):
req = Request("http://url", meta={"result": "ITSME"})
item = {"requests": req}
new_item = yield self.pipe.process_item(item, self.spider)
new_item = yield self.pipe.process_item(item)
assert new_item["results"] == [(True, "ITSME")]
assert self.pipe._mockcalled == [
"get_media_requests",
@ -555,12 +554,11 @@ class TestMediaFailedFailure:
settings = None
def setup_method(self):
spider_cls = Spider
self.spider = spider_cls("media.com")
crawler = get_crawler(spider_cls, self.settings)
crawler = get_crawler(DefaultSpider, self.settings)
crawler.spider = crawler._create_spider()
self.pipe = self.pipeline_class.from_crawler(crawler)
self.pipe.download_func = _mocked_download_func
self.pipe.open_spider(self.spider)
self.pipe.open_spider()
self.info = self.pipe.spiderinfo
self.fingerprint = crawler.request_fingerprinter.fingerprint
@ -587,7 +585,7 @@ class TestMediaFailedFailure:
with pytest.warns(
ScrapyDeprecationWarning, match="media_failed returned a Failure instance"
):
new_item = yield self.pipe.process_item(item, self.spider)
new_item = yield self.pipe.process_item(item)
assert len(new_item["results"]) == 1
assert new_item["results"][0][0] is False
assert isinstance(new_item["results"][0][1], Failure)

157
tests/test_pipelines.py
View File

@ -4,6 +4,7 @@ import pytest
from twisted.internet.defer import Deferred, inlineCallbacks, succeed
from scrapy import Request, Spider, signals
from scrapy.crawler import Crawler
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.pipelines import ItemPipelineManager
from scrapy.utils.asyncio import call_later
@ -19,17 +20,25 @@ from tests.mockserver.http import MockServer
class SimplePipeline:
def process_item(self, item, spider):
def process_item(self, item):
item["pipeline_passed"] = True
return item
class DeprecatedSpiderArgPipeline:
def open_spider(self, spider):
pass
def close_spider(self, spider):
pass
class DeferredPipeline:
def cb(self, item):
item["pipeline_passed"] = True
return item
def process_item(self, item, spider):
def process_item(self, item):
d = Deferred()
d.addCallback(self.cb)
d.callback(item)
@ -37,7 +46,7 @@ class DeferredPipeline:
class AsyncDefPipeline:
async def process_item(self, item, spider):
async def process_item(self, item):
d = Deferred()
call_later(0, d.callback, None)
await maybe_deferred_to_future(d)
@ -46,7 +55,7 @@ class AsyncDefPipeline:
class AsyncDefAsyncioPipeline:
async def process_item(self, item, spider):
async def process_item(self, item):
d = Deferred()
loop = asyncio.get_event_loop()
loop.call_later(0, d.callback, None)
@ -57,7 +66,7 @@ class AsyncDefAsyncioPipeline:
class AsyncDefNotAsyncioPipeline:
async def process_item(self, item, spider):
async def process_item(self, item):
d1 = Deferred()
from twisted.internet import reactor
@ -184,6 +193,9 @@ class TestCustomPipelineManager:
def open_spider(self, spider): # pylint: disable=signature-differs
return super().open_spider(spider)
def close_spider(self, spider): # pylint: disable=signature-differs
return super().close_spider(spider)
def process_item(self, item, spider): # pylint: disable=signature-differs
with pytest.warns(
ScrapyDeprecationWarning,
@ -212,6 +224,10 @@ class TestCustomPipelineManager:
ScrapyDeprecationWarning,
match=r"The open_spider\(\) method of .+\.CustomPipelineManager requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"The close_spider\(\) method of .+\.CustomPipelineManager requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"The process_item\(\) method of .+\.CustomPipelineManager requires a spider argument",
@ -220,6 +236,10 @@ class TestCustomPipelineManager:
ScrapyDeprecationWarning,
match=r"Passing a spider argument to CustomPipelineManager.open_spider\(\) is deprecated",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to CustomPipelineManager.close_spider\(\) is deprecated",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager overrides process_item\(\) but doesn't override process_item_async\(\)",
@ -254,7 +274,7 @@ class TestCustomPipelineManager:
def process_item(self, item, spider):
for pipeline in self.pipelines:
item = pipeline.process_item(item, spider)
item = pipeline.process_item(item)
return succeed(item)
items = []
@ -294,3 +314,128 @@ class TestCustomPipelineManager:
await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
assert len(items) == 1
class TestMiddlewareManagerSpider:
"""Tests for the deprecated spider arg handling in MiddlewareManager.
Here because MiddlewareManager doesn't have methods that could take a spider arg."""
@pytest.fixture
def crawler(self) -> Crawler:
return get_crawler(Spider)
def test_deprecated_spider_arg_no_crawler_spider(self, crawler: Crawler) -> None:
"""Crawler is provided, but doesn't have a spider. The instance passed to the method is
ignored and raises a warning."""
mwman = ItemPipelineManager(crawler=crawler)
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",
),
):
mwman._add_middleware(DeprecatedSpiderArgPipeline())
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated",
),
pytest.raises(
ValueError,
match="ItemPipelineManager needs to access self.crawler.spider but it is None",
),
):
mwman.open_spider(DefaultSpider())
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated",
),
pytest.raises(
ValueError,
match="ItemPipelineManager needs to access self.crawler.spider but it is None",
),
):
mwman.close_spider(DefaultSpider())
def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None:
"""Crawler is provided and has a spider, works. The instance passed to the method is ignored,
even if mismatched, but raises a warning."""
mwman = ItemPipelineManager(crawler=crawler)
crawler.spider = crawler._create_spider("foo")
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated",
):
mwman.open_spider(DefaultSpider())
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated",
):
mwman.close_spider(DefaultSpider())
def test_deprecated_spider_arg_without_crawler(self) -> None:
"""The first instance passed to the method is used, with a warning. Mismatched ones raise an error."""
with pytest.warns(
ScrapyDeprecationWarning,
match="was called without the crawler argument",
):
mwman = ItemPipelineManager()
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",
),
):
mwman._add_middleware(DeprecatedSpiderArgPipeline())
with pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to ItemPipelineManager.open_spider\(\) is deprecated",
):
mwman.open_spider(DefaultSpider())
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"Passing a spider argument to ItemPipelineManager.close_spider\(\) is deprecated",
),
pytest.raises(
RuntimeError, match="Different instances of Spider were passed"
),
):
mwman.close_spider(DefaultSpider())
mwman.close_spider()
def test_no_spider_arg_without_crawler(self) -> None:
"""If no crawler and no spider arg, raise an error."""
with pytest.warns(
ScrapyDeprecationWarning,
match="was called without the crawler argument",
):
mwman = ItemPipelineManager()
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",
),
):
mwman._add_middleware(DeprecatedSpiderArgPipeline())
with (
pytest.raises(
ValueError,
match="has no known Spider instance",
),
):
mwman.open_spider()

4
tests/test_request_cb_kwargs.py
View File

@ -33,12 +33,12 @@ class InjectArgumentsSpiderMiddleware:
request.cb_kwargs["from_process_start"] = True
yield request
def process_spider_input(self, response, spider):
def process_spider_input(self, response):
request = response.request
if request.callback.__name__ == "parse_spider_mw":
request.cb_kwargs["from_process_spider_input"] = True
def process_spider_output(self, response, result, spider):
def process_spider_output(self, response, result):
for element in result:
if (
isinstance(element, Request)

4
tests/test_settings/__init__.py
View File

@ -451,7 +451,7 @@ class TestSettings:
def test_passing_objects_as_values(self):
class TestPipeline:
def process_item(self, i, s):
def process_item(self, i):
return i
settings = Settings(
@ -471,7 +471,7 @@ class TestSettings:
assert priority == 800
assert mypipeline == TestPipeline
assert isinstance(mypipeline(), TestPipeline)
assert mypipeline().process_item("item", None) == "item"
assert mypipeline().process_item("item") == "item"
myhandler = settings.getdict("DOWNLOAD_HANDLERS").pop("ftp")
assert myhandler == FileDownloadHandler

71
tests/test_spidermiddleware.py
View File

@ -10,7 +10,7 @@ from testfixtures import LogCapture
from twisted.internet import defer
from scrapy.core.spidermw import SpiderMiddlewareManager
from scrapy.exceptions import _InvalidOutput
from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.asyncgen import collect_asyncgen
@ -54,7 +54,7 @@ class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_invalid_process_spider_input(self):
class InvalidProcessSpiderInputMiddleware:
def process_spider_input(self, response, spider):
def process_spider_input(self, response):
return 1
self.mwman._add_middleware(InvalidProcessSpiderInputMiddleware())
@ -68,7 +68,7 @@ class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_invalid_process_spider_output(self):
class InvalidProcessSpiderOutputMiddleware:
def process_spider_output(self, response, result, spider):
def process_spider_output(self, response, result):
return 1
self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())
@ -82,11 +82,11 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_invalid_process_spider_exception(self):
class InvalidProcessSpiderOutputExceptionMiddleware:
def process_spider_exception(self, response, exception, spider):
def process_spider_exception(self, response, exception):
return 1
class RaiseExceptionProcessSpiderOutputMiddleware:
def process_spider_output(self, response, result, spider):
def process_spider_output(self, response, result):
raise RuntimeError
self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())
@ -101,11 +101,11 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_process_spider_exception_return_none(self):
class ProcessSpiderExceptionReturnNoneMiddleware:
def process_spider_exception(self, response, exception, spider):
def process_spider_exception(self, response, exception):
return None
class RaiseExceptionProcessSpiderOutputMiddleware:
def process_spider_output(self, response, result, spider):
def process_spider_output(self, response, result):
1 / 0
self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
@ -191,34 +191,34 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
class ProcessSpiderOutputSimpleMiddleware:
def process_spider_output(self, response, result, spider):
def process_spider_output(self, response, result):
yield from result
class ProcessSpiderOutputAsyncGenMiddleware:
async def process_spider_output(self, response, result, spider):
async def process_spider_output(self, response, result):
async for r in result:
yield r
class ProcessSpiderOutputUniversalMiddleware:
def process_spider_output(self, response, result, spider):
def process_spider_output(self, response, result):
yield from result
async def process_spider_output_async(self, response, result, spider):
async def process_spider_output_async(self, response, result):
async for r in result:
yield r
class ProcessSpiderExceptionSimpleIterableMiddleware:
def process_spider_exception(self, response, exception, spider):
def process_spider_exception(self, response, exception):
yield {"foo": 1}
yield {"foo": 2}
yield {"foo": 3}
class ProcessSpiderExceptionAsyncIteratorMiddleware:
async def process_spider_exception(self, response, exception, spider):
async def process_spider_exception(self, response, exception):
yield {"foo": 1}
d = defer.Deferred()
call_later(0, d.callback, None)
@ -315,12 +315,12 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
class ProcessSpiderOutputNonIterableMiddleware:
def process_spider_output(self, response, result, spider):
def process_spider_output(self, response, result):
return
class ProcessSpiderOutputCoroutineMiddleware:
async def process_spider_output(self, response, result, spider):
async def process_spider_output(self, response, result):
return result
@ -384,23 +384,23 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):
class UniversalMiddlewareNoSync:
async def process_spider_output_async(self, response, result, spider):
async def process_spider_output_async(self, response, result):
yield
class UniversalMiddlewareBothSync:
def process_spider_output(self, response, result, spider):
def process_spider_output(self, response, result):
yield
def process_spider_output_async(self, response, result, spider):
def process_spider_output_async(self, response, result):
yield
class UniversalMiddlewareBothAsync:
async def process_spider_output(self, response, result, spider):
async def process_spider_output(self, response, result):
yield
async def process_spider_output_async(self, response, result, spider):
async def process_spider_output_async(self, response, result):
yield
@ -600,3 +600,34 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
async def test_exc_async_simple(self):
"""Async exc mw -> simple output mw; cannot work as downgrading is not supported"""
await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN)
class TestDeprecatedSpiderArg(TestSpiderMiddleware):
@deferred_f_from_coro_f
async def test_deprecated_spider_arg(self):
class DeprecatedSpiderArgMiddleware:
def process_spider_input(self, response, spider):
return None
def process_spider_output(self, response, result, spider):
1 / 0
def process_spider_exception(self, response, exception, spider):
return []
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"process_spider_input\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"process_spider_output\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"process_spider_exception\(\) requires a spider argument",
),
):
self.mwman._add_middleware(DeprecatedSpiderArgMiddleware())
await self._scrape_response()

14
tests/test_spidermiddleware_depth.py
View File

@ -21,12 +21,6 @@ def crawler() -> Crawler:
return get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True})
@pytest.fixture
def spider(crawler: Crawler) -> Spider:
crawler.spider = crawler._create_spider("scrapytest.org")
return crawler.spider
@pytest.fixture
def stats(crawler: Crawler) -> Generator[StatsCollector]:
assert crawler.stats is not None
@ -42,15 +36,13 @@ def mw(crawler: Crawler) -> DepthMiddleware:
return DepthMiddleware.from_crawler(crawler)
def test_process_spider_output(
mw: DepthMiddleware, stats: StatsCollector, spider: Spider
) -> None:
def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> None:
req = Request("http://scrapytest.org")
resp = Response("http://scrapytest.org")
resp.request = req
result = [Request("http://scrapytest.org")]
out = list(mw.process_spider_output(resp, result, spider))
out = list(mw.process_spider_output(resp, result))
assert out == result
rdc = stats.get_value("request_depth_count/1")
@ -58,7 +50,7 @@ def test_process_spider_output(
req.meta["depth"] = 1
out2 = list(mw.process_spider_output(resp, result, spider))
out2 = list(mw.process_spider_output(resp, result))
assert not out2
rdm = stats.get_value("request_depth_max")

95
tests/test_spidermiddleware_httperror.py
View File

@ -7,9 +7,8 @@ from testfixtures import LogCapture
from twisted.internet.defer import inlineCallbacks
from scrapy.http import Request, Response
from scrapy.settings import Settings
from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware
from scrapy.spiders import Spider
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import MockServerSpider
@ -57,12 +56,6 @@ def _response(request: Request, status_code: int) -> Response:
return Response(request.url, status=status_code, request=request)
@pytest.fixture
def spider() -> Spider:
crawler = get_crawler(Spider)
return Spider.from_crawler(crawler, name="foo")
@pytest.fixture
def res200() -> Response:
return _response(req, 200)
@ -81,36 +74,35 @@ def res404() -> Response:
class TestHttpErrorMiddleware:
@pytest.fixture
def mw(self) -> HttpErrorMiddleware:
return HttpErrorMiddleware(Settings({}))
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
return HttpErrorMiddleware.from_crawler(crawler)
def test_process_spider_input(
self,
mw: HttpErrorMiddleware,
spider: Spider,
res200: Response,
res404: Response,
self, mw: HttpErrorMiddleware, res200: Response, res404: Response
) -> None:
mw.process_spider_input(res200, spider)
mw.process_spider_input(res200)
with pytest.raises(HttpError):
mw.process_spider_input(res404, spider)
mw.process_spider_input(res404)
def test_process_spider_exception(
self, mw: HttpErrorMiddleware, spider: Spider, res404: Response
self, mw: HttpErrorMiddleware, res404: Response
) -> None:
assert mw.process_spider_exception(res404, HttpError(res404), spider) == []
assert mw.process_spider_exception(res404, Exception(), spider) is None
assert mw.process_spider_exception(res404, HttpError(res404)) == []
assert mw.process_spider_exception(res404, Exception()) is None
def test_handle_httpstatus_list(
self, mw: HttpErrorMiddleware, spider: Spider, res404: Response
self, mw: HttpErrorMiddleware, res404: Response
) -> None:
request = Request(
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
)
res = _response(request, 404)
mw.process_spider_input(res, spider)
mw.process_spider_input(res)
spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
mw.process_spider_input(res404, spider)
assert mw.crawler.spider
mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
mw.process_spider_input(res404)
class TestHttpErrorMiddlewareSettings:
@ -118,76 +110,71 @@ class TestHttpErrorMiddlewareSettings:
@pytest.fixture
def mw(self) -> HttpErrorMiddleware:
return HttpErrorMiddleware(Settings({"HTTPERROR_ALLOWED_CODES": (402,)}))
crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOWED_CODES": (402,)})
crawler.spider = crawler._create_spider()
return HttpErrorMiddleware.from_crawler(crawler)
def test_process_spider_input(
self,
mw: HttpErrorMiddleware,
spider: Spider,
res200: Response,
res402: Response,
res404: Response,
) -> None:
mw.process_spider_input(res200, spider)
mw.process_spider_input(res200)
with pytest.raises(HttpError):
mw.process_spider_input(res404, spider)
mw.process_spider_input(res402, spider)
mw.process_spider_input(res404)
mw.process_spider_input(res402)
def test_meta_overrides_settings(
self, mw: HttpErrorMiddleware, spider: Spider
) -> None:
def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None:
request = Request(
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
)
res404 = _response(request, 404)
res402 = _response(request, 402)
mw.process_spider_input(res404, spider)
mw.process_spider_input(res404)
with pytest.raises(HttpError):
mw.process_spider_input(res402, spider)
mw.process_spider_input(res402)
def test_spider_override_settings(
self,
mw: HttpErrorMiddleware,
spider: Spider,
res402: Response,
res404: Response,
self, mw: HttpErrorMiddleware, res402: Response, res404: Response
) -> None:
spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
mw.process_spider_input(res404, spider)
assert mw.crawler.spider
mw.crawler.spider.handle_httpstatus_list = [404] # type: ignore[attr-defined]
mw.process_spider_input(res404)
with pytest.raises(HttpError):
mw.process_spider_input(res402, spider)
mw.process_spider_input(res402)
class TestHttpErrorMiddlewareHandleAll:
@pytest.fixture
def mw(self) -> HttpErrorMiddleware:
return HttpErrorMiddleware(Settings({"HTTPERROR_ALLOW_ALL": True}))
crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOW_ALL": True})
crawler.spider = crawler._create_spider()
return HttpErrorMiddleware.from_crawler(crawler)
def test_process_spider_input(
self,
mw: HttpErrorMiddleware,
spider: Spider,
res200: Response,
res404: Response,
) -> None:
mw.process_spider_input(res200, spider)
mw.process_spider_input(res404, spider)
mw.process_spider_input(res200)
mw.process_spider_input(res404)
def test_meta_overrides_settings(
self, mw: HttpErrorMiddleware, spider: Spider
) -> None:
def test_meta_overrides_settings(self, mw: HttpErrorMiddleware) -> None:
request = Request(
"http://scrapytest.org", meta={"handle_httpstatus_list": [404]}
)
res404 = _response(request, 404)
res402 = _response(request, 402)
mw.process_spider_input(res404, spider)
mw.process_spider_input(res404)
with pytest.raises(HttpError):
mw.process_spider_input(res402, spider)
mw.process_spider_input(res402)
def test_httperror_allow_all_false(self, spider: Spider) -> None:
def test_httperror_allow_all_false(self) -> None:
crawler = get_crawler(_HttpErrorSpider)
mw = HttpErrorMiddleware.from_crawler(crawler)
request_httpstatus_false = Request(
@ -200,8 +187,8 @@ class TestHttpErrorMiddlewareHandleAll:
res402 = _response(request_httpstatus_true, 402)
with pytest.raises(HttpError):
mw.process_spider_input(res404, spider)
mw.process_spider_input(res402, spider)
mw.process_spider_input(res404)
mw.process_spider_input(res402)
class TestHttpErrorMiddlewareIntegrational:
@ -218,7 +205,7 @@ class TestHttpErrorMiddlewareIntegrational:
def test_middleware_works(self):
crawler = get_crawler(_HttpErrorSpider)
yield crawler.crawl(mockserver=self.mockserver)
assert not crawler.spider.skipped, crawler.spider.skipped
assert not crawler.spider.skipped
assert crawler.spider.parsed == {"200"}
assert crawler.spider.failed == {"404", "402", "500"}

87
tests/test_spidermiddleware_output_chain.py
View File

@ -6,18 +6,27 @@ from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
class LogExceptionMiddleware:
def process_spider_exception(self, response, exception, spider):
spider.logger.info(
class _BaseSpiderMiddleware:
def __init__(self, crawler):
self.crawler = crawler
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
class LogExceptionMiddleware(_BaseSpiderMiddleware):
def process_spider_exception(self, response, exception):
self.crawler.spider.logger.info(
"Middleware: %s exception caught", exception.__class__.__name__
)
# ================================================================================
# (0) recover from an exception on a spider callback
class RecoveryMiddleware:
def process_spider_exception(self, response, exception, spider):
spider.logger.info(
class RecoveryMiddleware(_BaseSpiderMiddleware):
def process_spider_exception(self, response, exception):
self.crawler.spider.logger.info(
"Middleware: %s exception caught", exception.__class__.__name__
)
return [
@ -55,9 +64,9 @@ class RecoveryAsyncGenSpider(RecoverySpider):
# ================================================================================
# (1) exceptions from a spider middleware's process_spider_input method
class FailProcessSpiderInputMiddleware:
def process_spider_input(self, response, spider):
spider.logger.info("Middleware: will raise IndexError")
class FailProcessSpiderInputMiddleware(_BaseSpiderMiddleware):
def process_spider_input(self, response):
self.crawler.spider.logger.info("Middleware: will raise IndexError")
raise IndexError
@ -159,27 +168,31 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS
# ================================================================================
# (4) exceptions from a middleware process_spider_output method (generator)
class _GeneratorDoNothingMiddleware:
def process_spider_output(self, response, result, spider):
class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware):
def process_spider_output(self, response, result):
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r
def process_spider_exception(self, response, exception, spider):
def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
self.crawler.spider.logger.info(
"%s: %s caught", method, exception.__class__.__name__
)
class GeneratorFailMiddleware:
def process_spider_output(self, response, result, spider):
class GeneratorFailMiddleware(_BaseSpiderMiddleware):
def process_spider_output(self, response, result):
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r
raise LookupError
def process_spider_exception(self, response, exception, spider):
def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
self.crawler.spider.logger.info(
"%s: %s caught", method, exception.__class__.__name__
)
yield {"processed": [method]}
@ -187,15 +200,17 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware):
pass
class GeneratorRecoverMiddleware:
def process_spider_output(self, response, result, spider):
class GeneratorRecoverMiddleware(_BaseSpiderMiddleware):
def process_spider_output(self, response, result):
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
yield r
def process_spider_exception(self, response, exception, spider):
def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
self.crawler.spider.logger.info(
"%s: %s caught", method, exception.__class__.__name__
)
yield {"processed": [method]}
@ -226,30 +241,34 @@ class GeneratorOutputChainSpider(Spider):
# (5) exceptions from a middleware process_spider_output method (not generator)
class _NotGeneratorDoNothingMiddleware:
def process_spider_output(self, response, result, spider):
class _NotGeneratorDoNothingMiddleware(_BaseSpiderMiddleware):
def process_spider_output(self, response, result):
out = []
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r)
return out
def process_spider_exception(self, response, exception, spider):
def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
self.crawler.spider.logger.info(
"%s: %s caught", method, exception.__class__.__name__
)
class NotGeneratorFailMiddleware:
def process_spider_output(self, response, result, spider):
class NotGeneratorFailMiddleware(_BaseSpiderMiddleware):
def process_spider_output(self, response, result):
out = []
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r)
raise ReferenceError
def process_spider_exception(self, response, exception, spider):
def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
self.crawler.spider.logger.info(
"%s: %s caught", method, exception.__class__.__name__
)
return [{"processed": [method]}]
@ -257,17 +276,19 @@ class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddlewa
pass
class NotGeneratorRecoverMiddleware:
def process_spider_output(self, response, result, spider):
class NotGeneratorRecoverMiddleware(_BaseSpiderMiddleware):
def process_spider_output(self, response, result):
out = []
for r in result:
r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
out.append(r)
return out
def process_spider_exception(self, response, exception, spider):
def process_spider_exception(self, response, exception):
method = f"{self.__class__.__name__}.process_spider_exception"
spider.logger.info("%s: %s caught", method, exception.__class__.__name__)
self.crawler.spider.logger.info(
"%s: %s caught", method, exception.__class__.__name__
)
return [{"processed": [method]}]

6
tests/test_spidermiddleware_referer.py
View File

@ -61,11 +61,11 @@ class TestRefererMiddleware:
def get_response(self, origin: str) -> Response:
return Response(origin, headers=self.resp_headers)
def test(self, mw: RefererMiddleware, spider: Spider) -> None:
def test(self, mw: RefererMiddleware) -> None:
for origin, target, referrer in self.scenarii:
response = self.get_response(origin)
request = self.get_request(target)
out = list(mw.process_spider_output(response, [request], spider))
out = list(mw.process_spider_output(response, [request]))
assert out[0].headers.get("Referer") == referrer
@ -1036,7 +1036,7 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
response = self.get_response(parent)
request = self.get_request(target)
out = list(referrermw.process_spider_output(response, [request], spider))
out = list(referrermw.process_spider_output(response, [request]))
assert out[0].headers.get("Referer") == init_referrer
for status, url in redirections:

20
tests/test_spidermiddleware_urllength.py
View File

@ -27,11 +27,6 @@ def crawler() -> Crawler:
return get_crawler(Spider, {"URLLENGTH_LIMIT": maxlength})
@pytest.fixture
def spider(crawler: Crawler) -> Spider:
return crawler._create_spider("foo")
@pytest.fixture
def stats(crawler: Crawler) -> StatsCollector:
assert crawler.stats is not None
@ -43,22 +38,19 @@ def mw(crawler: Crawler) -> UrlLengthMiddleware:
return UrlLengthMiddleware.from_crawler(crawler)
def process_spider_output(mw: UrlLengthMiddleware, spider: Spider) -> list[Request]:
return list(mw.process_spider_output(response, reqs, spider))
def process_spider_output(mw: UrlLengthMiddleware) -> list[Request]:
return list(mw.process_spider_output(response, reqs))
def test_middleware_works(mw: UrlLengthMiddleware, spider: Spider) -> None:
assert process_spider_output(mw, spider) == [short_url_req]
def test_middleware_works(mw: UrlLengthMiddleware) -> None:
assert process_spider_output(mw) == [short_url_req]
def test_logging(
stats: StatsCollector,
mw: UrlLengthMiddleware,
spider: Spider,
caplog: pytest.LogCaptureFixture,
stats: StatsCollector, mw: UrlLengthMiddleware, caplog: pytest.LogCaptureFixture
) -> None:
with caplog.at_level(INFO):
process_spider_output(mw, spider)
process_spider_output(mw)
ric = stats.get_value("urllength/request_ignored_count")
assert ric == 1
assert f"Ignoring link (url length > {maxlength})" in caplog.text

22
tests/test_utils_defer.py
View File

@ -10,7 +10,6 @@ from twisted.internet.defer import Deferred, inlineCallbacks, succeed
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import (
_process_chain,
aiter_errback,
deferred_f_from_coro_f,
deferred_from_coro,
@ -19,7 +18,6 @@ from scrapy.utils.defer import (
maybe_deferred_to_future,
mustbe_deferred,
parallel_async,
process_parallel,
)
if TYPE_CHECKING:
@ -85,26 +83,6 @@ def eb1(failure, arg1, arg2):
return f"(eb1 {failure.value.__class__.__name__} {arg1} {arg2})"
class TestDeferUtils:
@deferred_f_from_coro_f
async def test_process_chain(self):
x = await _process_chain([cb1, cb2, cb3], "res", "v1", "v2")
assert x == "(cb3 (cb2 (cb1 res v1 v2) v1 v2) v1 v2)"
with pytest.raises(TypeError):
await _process_chain([cb1, cb_fail, cb3], "res", "v1", "v2")
@inlineCallbacks
def test_process_parallel(self):
x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2")
assert x == ["(cb1 res v1 v2)", "(cb2 res v1 v2)", "(cb3 res v1 v2)"]
@inlineCallbacks
def test_process_parallel_failure(self):
with pytest.raises(TypeError):
yield process_parallel([cb1, cb_fail, cb3], "res", "v1", "v2")
class TestIterErrback:
def test_iter_errback_good(self):
def itergood() -> Generator[int, None, None]: