diff --git a/docs/news.rst b/docs/news.rst index 9a68f8852..9f476ee21 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,23 @@ Release notes ============= +.. _release-VERSION: + +Scrapy VERSION (unreleased) +--------------------------- + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The ``from_settings()`` method of + :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is removed + without a deprecation period (this was needed because after the + introduction of the + :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and + switching built-in spider middlewares to it those middlewares need the + :class:`~scrapy.crawler.Crawler` instance at run time). Please use + ``from_crawler()`` instead. + .. _release-2.12.0: Scrapy 2.12.0 (2024-11-18) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 567a875b6..2211a822f 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -189,6 +189,19 @@ one or more of these methods: :param spider: the spider which raised the exception :type spider: :class:`~scrapy.Spider` object +Base class for custom spider middlewares +---------------------------------------- + +Scrapy provides a base class for custom spider middlewares. It's not required +to use it but it can help with simplifying middleware implementations and +reducing the amount of boilerplate code in :ref:`universal middlewares +`. + +.. module:: scrapy.spidermiddlewares.base + +.. autoclass:: BaseSpiderMiddleware + :members: + .. _topics-spider-middleware-ref: Built-in spider middleware reference diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py new file mode 100644 index 000000000..650192095 --- /dev/null +++ b/scrapy/spidermiddlewares/base.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from scrapy import Request, Spider + +if TYPE_CHECKING: + from collections.abc import AsyncIterable, Iterable + + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.http import Response + + +class BaseSpiderMiddleware: + """Optional base class for spider middlewares. + + This class provides helper methods for asynchronous ``process_spider_output`` + methods. Middlewares that don't have a ``process_spider_output`` method don't need + to use it. + + You can override the + :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_request` + method to add processing code for requests and the + :meth:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware.get_processed_item` + method to add processing code for items. These methods take a single + request or item from the spider output iterable and return a request or + item (the same or a new one), or ``None`` to remove this request or item + from the processing. + """ + + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + + def process_spider_output( + self, response: Response, result: Iterable[Any], spider: Spider + ) -> Iterable[Any]: + for o in result: + if isinstance(o, Request): + o = self.get_processed_request(o, response) + else: + o = self.get_processed_item(o, response) + if o is not None: + yield o + + async def process_spider_output_async( + self, response: Response, result: AsyncIterable[Any], spider: Spider + ) -> AsyncIterable[Any]: + async for o in result: + if isinstance(o, Request): + o = self.get_processed_request(o, response) + else: + o = self.get_processed_item(o, response) + if o is not None: + yield o + + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + """Return a processed request from the spider output. + + This method is called with a single request from the spider output. + It should return the same or a different request, or ``None`` to + ignore it. + + :param request: the input request + :type request: :class:`~scrapy.Request` object + + :param response: the response being processed + :type response: :class:`~scrapy.http.Response` object + + :return: the processed request or ``None`` + """ + return request + + def get_processed_item(self, item: Any, response: Response) -> Any: + """Return a processed item from the spider output. + + This method is called with a single item from the spider output. + It should return the same or a different item, or ``None`` to + ignore it. + + :param item: the input item + :type item: item object + + :param response: the response being processed + :type response: :class:`~scrapy.http.Response` object + + :return: the processed item or ``None`` + """ + return item diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py index 3164c1c03..65905f483 100644 --- a/scrapy/spidermiddlewares/depth.py +++ b/scrapy/spidermiddlewares/depth.py @@ -9,7 +9,7 @@ from __future__ import annotations import logging from typing import TYPE_CHECKING, Any -from scrapy.http import Request, Response +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware if TYPE_CHECKING: from collections.abc import AsyncIterable, Iterable @@ -19,14 +19,17 @@ if TYPE_CHECKING: from scrapy import Spider from scrapy.crawler import Crawler + from scrapy.http import Request, Response from scrapy.statscollectors import StatsCollector logger = logging.getLogger(__name__) -class DepthMiddleware: - def __init__( +class DepthMiddleware(BaseSpiderMiddleware): + crawler: Crawler + + def __init__( # pylint: disable=super-init-not-called self, maxdepth: int, stats: StatsCollector, @@ -45,21 +48,22 @@ class DepthMiddleware: verbose = settings.getbool("DEPTH_STATS_VERBOSE") prio = settings.getint("DEPTH_PRIORITY") assert crawler.stats - return cls(maxdepth, crawler.stats, verbose, prio) + o = cls(maxdepth, crawler.stats, verbose, prio) + o.crawler = crawler + return o def process_spider_output( self, response: Response, result: Iterable[Any], spider: Spider ) -> Iterable[Any]: self._init_depth(response, spider) - return (r for r in result if self._filter(r, response, spider)) + yield from super().process_spider_output(response, result, spider) async def process_spider_output_async( self, response: Response, result: AsyncIterable[Any], spider: Spider ) -> AsyncIterable[Any]: self._init_depth(response, spider) - async for r in result: - if self._filter(r, response, spider): - yield r + async for o in super().process_spider_output_async(response, result, spider): + yield o def _init_depth(self, response: Response, spider: Spider) -> None: # base case (depth=0) @@ -68,9 +72,9 @@ class DepthMiddleware: if self.verbose_stats: self.stats.inc_value("request_depth_count/0", spider=spider) - def _filter(self, request: Any, response: Response, spider: Spider) -> bool: - if not isinstance(request, Request): - return True + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: depth = response.meta["depth"] + 1 request.meta["depth"] = depth if self.prio: @@ -79,10 +83,12 @@ class DepthMiddleware: logger.debug( "Ignoring link (depth > %(maxdepth)d): %(requrl)s ", {"maxdepth": self.maxdepth, "requrl": request.url}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) - return False + return None if self.verbose_stats: - self.stats.inc_value(f"request_depth_count/{depth}", spider=spider) - self.stats.max_value("request_depth_max", depth, spider=spider) - return True + self.stats.inc_value( + f"request_depth_count/{depth}", spider=self.crawler.spider + ) + self.stats.max_value("request_depth_max", depth, spider=self.crawler.spider) + return request diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py index 646beb911..0918c9fac 100644 --- a/scrapy/spidermiddlewares/offsite.py +++ b/scrapy/spidermiddlewares/offsite.py @@ -9,11 +9,11 @@ from __future__ import annotations import logging import re import warnings -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING from scrapy import Spider, signals from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.http import Request, Response +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware from scrapy.utils.httpobj import urlparse_cached warnings.warn( @@ -23,61 +23,52 @@ warnings.warn( ) if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable - # typing.Self requires Python 3.11 from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http import Request, Response from scrapy.statscollectors import StatsCollector logger = logging.getLogger(__name__) -class OffsiteMiddleware: - def __init__(self, stats: StatsCollector): +class OffsiteMiddleware(BaseSpiderMiddleware): + crawler: Crawler + + def __init__(self, stats: StatsCollector): # pylint: disable=super-init-not-called self.stats: StatsCollector = stats @classmethod def from_crawler(cls, crawler: Crawler) -> Self: assert crawler.stats o = cls(crawler.stats) + o.crawler = crawler crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) return o - def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - return (r for r in result if self._filter(r, spider)) - - async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: - async for r in result: - if self._filter(r, spider): - yield r - - def _filter(self, request: Any, spider: Spider) -> bool: - if not isinstance(request, Request): - return True + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + assert self.crawler.spider if ( request.dont_filter or request.meta.get("allow_offsite") - or self.should_follow(request, spider) + or self.should_follow(request, self.crawler.spider) ): - return True + return request domain = urlparse_cached(request).hostname if domain and domain not in self.domains_seen: self.domains_seen.add(domain) logger.debug( "Filtered offsite request to %(domain)r: %(request)s", {"domain": domain, "request": request}, - extra={"spider": spider}, + extra={"spider": self.crawler.spider}, ) - self.stats.inc_value("offsite/domains", spider=spider) - self.stats.inc_value("offsite/filtered", spider=spider) - return False + self.stats.inc_value("offsite/domains", spider=self.crawler.spider) + self.stats.inc_value("offsite/filtered", spider=self.crawler.spider) + return None def should_follow(self, request: Request, spider: Spider) -> bool: regex = self.host_regex diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index a3a1e5b92..b2ba8ba8c 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -6,7 +6,7 @@ originated it. from __future__ import annotations import warnings -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, cast from urllib.parse import urlparse from w3lib.url import safe_url_string @@ -14,13 +14,12 @@ from w3lib.url import safe_url_string from scrapy import Spider, signals from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware from scrapy.utils.misc import load_object from scrapy.utils.python import to_unicode from scrapy.utils.url import strip_url if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable - # typing.Self requires Python 3.11 from typing_extensions import Self @@ -327,8 +326,8 @@ def _load_policy_class( return None -class RefererMiddleware: - def __init__(self, settings: BaseSettings | None = None): +class RefererMiddleware(BaseSpiderMiddleware): + def __init__(self, settings: BaseSettings | None = None): # pylint: disable=super-init-not-called self.default_policy: type[ReferrerPolicy] = DefaultReferrerPolicy if settings is not None: settings_policy = _load_policy_class(settings.get("REFERRER_POLICY")) @@ -370,23 +369,13 @@ class RefererMiddleware: cls = _load_policy_class(policy_name, warning_only=True) return cls() if cls else self.default_policy() - def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - return (self._set_referer(r, response) for r in result) - - async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: - async for r in result: - yield self._set_referer(r, response) - - def _set_referer(self, r: Any, response: Response) -> Any: - if isinstance(r, Request): - referrer = self.policy(response, r).referrer(response.url, r.url) - if referrer is not None: - r.headers.setdefault("Referer", referrer) - return r + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + referrer = self.policy(response, request).referrer(response.url, request.url) + if referrer is not None: + request.headers.setdefault("Referer", referrer) + return request def request_scheduled(self, request: Request, spider: Spider) -> None: # check redirected request to patch "Referer" header if necessary diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py index a1cd1bb7c..177c19e1b 100644 --- a/scrapy/spidermiddlewares/urllength.py +++ b/scrapy/spidermiddlewares/urllength.py @@ -7,72 +7,49 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -import warnings -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning -from scrapy.http import Request, Response +from scrapy.exceptions import NotConfigured +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable - # typing.Self requires Python 3.11 from typing_extensions import Self - from scrapy import Spider from scrapy.crawler import Crawler - from scrapy.settings import BaseSettings + from scrapy.http import Request, Response logger = logging.getLogger(__name__) -class UrlLengthMiddleware: - def __init__(self, maxlength: int): +class UrlLengthMiddleware(BaseSpiderMiddleware): + crawler: Crawler + + def __init__(self, maxlength: int): # pylint: disable=super-init-not-called self.maxlength: int = maxlength - @classmethod - def from_settings(cls, settings: BaseSettings) -> Self: - warnings.warn( - f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.", - category=ScrapyDeprecationWarning, - stacklevel=2, - ) - return cls._from_settings(settings) - @classmethod def from_crawler(cls, crawler: Crawler) -> Self: - return cls._from_settings(crawler.settings) - - @classmethod - def _from_settings(cls, settings: BaseSettings) -> Self: - maxlength = settings.getint("URLLENGTH_LIMIT") + maxlength = crawler.settings.getint("URLLENGTH_LIMIT") if not maxlength: raise NotConfigured - return cls(maxlength) + o = cls(maxlength) + o.crawler = crawler + return o - def process_spider_output( - self, response: Response, result: Iterable[Any], spider: Spider - ) -> Iterable[Any]: - return (r for r in result if self._filter(r, spider)) - - async def process_spider_output_async( - self, response: Response, result: AsyncIterable[Any], spider: Spider - ) -> AsyncIterable[Any]: - async for r in result: - if self._filter(r, spider): - yield r - - def _filter(self, request: Any, spider: Spider) -> bool: - if isinstance(request, Request) and len(request.url) > self.maxlength: - logger.info( - "Ignoring link (url length > %(maxlength)d): %(url)s ", - {"maxlength": self.maxlength, "url": request.url}, - extra={"spider": spider}, - ) - assert spider.crawler.stats - spider.crawler.stats.inc_value( - "urllength/request_ignored_count", spider=spider - ) - return False - return True + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + if len(request.url) <= self.maxlength: + return request + logger.info( + "Ignoring link (url length > %(maxlength)d): %(url)s ", + {"maxlength": self.maxlength, "url": request.url}, + extra={"spider": self.crawler.spider}, + ) + assert self.crawler.stats + self.crawler.stats.inc_value( + "urllength/request_ignored_count", spider=self.crawler.spider + ) + return None diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py new file mode 100644 index 000000000..46be879f3 --- /dev/null +++ b/tests/test_spidermiddleware_base.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy import Request, Spider +from scrapy.http import Response +from scrapy.spidermiddlewares.base import BaseSpiderMiddleware +from scrapy.utils.test import get_crawler + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + + +@pytest.fixture +def crawler() -> Crawler: + return get_crawler(Spider) + + +def test_trivial(crawler): + class TrivialSpiderMiddleware(BaseSpiderMiddleware): + pass + + mw = TrivialSpiderMiddleware.from_crawler(crawler) + assert hasattr(mw, "crawler") + assert mw.crawler is crawler + test_req = Request("data:,") + spider_output = [test_req, {"foo": "bar"}] + processed = list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ) + assert processed == [test_req, {"foo": "bar"}] + + +def test_processed_request(crawler): + class ProcessReqSpiderMiddleware(BaseSpiderMiddleware): + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + if request.url == "data:2,": + return None + if request.url == "data:3,": + return Request("data:30,") + return request + + mw = ProcessReqSpiderMiddleware.from_crawler(crawler) + test_req1 = Request("data:1,") + test_req2 = Request("data:2,") + test_req3 = Request("data:3,") + spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3] + processed = list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ) + assert len(processed) == 3 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": "bar"} + assert isinstance(processed[2], Request) + assert processed[2].url == "data:30," + + +def test_processed_item(crawler): + class ProcessItemSpiderMiddleware(BaseSpiderMiddleware): + def get_processed_item(self, item: Any, response: Response) -> Any: + if item["foo"] == 2: + return None + if item["foo"] == 3: + item["foo"] = 30 + return item + + mw = ProcessItemSpiderMiddleware.from_crawler(crawler) + test_req = Request("data:,") + spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}] + processed = list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ) + assert processed == [{"foo": 1}, test_req, {"foo": 30}] + + +def test_processed_both(crawler): + class ProcessBothSpiderMiddleware(BaseSpiderMiddleware): + def get_processed_request( + self, request: Request, response: Response + ) -> Request | None: + if request.url == "data:2,": + return None + if request.url == "data:3,": + return Request("data:30,") + return request + + def get_processed_item(self, item: Any, response: Response) -> Any: + if item["foo"] == 2: + return None + if item["foo"] == 3: + item["foo"] = 30 + return item + + mw = ProcessBothSpiderMiddleware.from_crawler(crawler) + test_req1 = Request("data:1,") + test_req2 = Request("data:2,") + test_req3 = Request("data:3,") + spider_output = [ + test_req1, + {"foo": 1}, + {"foo": 2}, + test_req2, + {"foo": 3}, + test_req3, + ] + processed = list( + mw.process_spider_output(Response("data:,"), spider_output, crawler.spider) + ) + assert len(processed) == 4 + assert isinstance(processed[0], Request) + assert processed[0].url == "data:1," + assert processed[1] == {"foo": 1} + assert processed[2] == {"foo": 30} + assert isinstance(processed[3], Request) + assert processed[3].url == "data:30," diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py index dfcc141c3..9b4aa624c 100644 --- a/tests/test_spidermiddleware_depth.py +++ b/tests/test_spidermiddleware_depth.py @@ -1,19 +1,18 @@ from scrapy.http import Request, Response from scrapy.spidermiddlewares.depth import DepthMiddleware from scrapy.spiders import Spider -from scrapy.statscollectors import StatsCollector from scrapy.utils.test import get_crawler class TestDepthMiddleware: def setup_method(self): - crawler = get_crawler(Spider) + crawler = get_crawler(Spider, {"DEPTH_LIMIT": 1, "DEPTH_STATS_VERBOSE": True}) self.spider = crawler._create_spider("scrapytest.org") - self.stats = StatsCollector(crawler) + self.stats = crawler.stats self.stats.open_spider(self.spider) - self.mw = DepthMiddleware(1, self.stats, True) + self.mw = DepthMiddleware.from_crawler(crawler) def test_process_spider_output(self): req = Request("http://scrapytest.org") diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index f4563a0a4..e4f4b8f9b 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -10,7 +10,7 @@ from scrapy.utils.test import get_crawler class TestOffsiteMiddleware: def setup_method(self): crawler = get_crawler(Spider) - self.spider = crawler._create_spider(**self._get_spiderargs()) + self.spider = crawler.spider = crawler._create_spider(**self._get_spiderargs()) self.mw = OffsiteMiddleware.from_crawler(crawler) self.mw.spider_opened(self.spider)