From 15885a8db4683e974fbcbb56a484846108afc350 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 10 Aug 2026 17:55:10 +0200 Subject: [PATCH 1/2] Generalize the use of build_from_crawler() internally (#7808) * Generalize the use of build_from_crawler() internally * Do not use build_from_crawler() for spiders, since they are guaranteed having from_crawler() --- scrapy/core/downloader/__init__.py | 5 +- scrapy/core/scraper.py | 12 ++- scrapy/crawler.py | 4 +- scrapy/downloadermiddlewares/robotstxt.py | 6 +- scrapy/extensions/memusage.py | 3 +- scrapy/extensions/statsmailer.py | 3 +- tests/test_command_shell.py | 2 +- tests/test_downloader_handler_twisted_ftp.py | 2 +- .../test_downloader_handler_twisted_http11.py | 3 +- .../test_downloader_handler_twisted_http2.py | 3 +- tests/test_downloadermiddleware.py | 3 +- tests/test_downloadermiddleware_cookies.py | 21 +++--- ...est_downloadermiddleware_defaultheaders.py | 3 +- ...st_downloadermiddleware_downloadtimeout.py | 3 +- tests/test_downloadermiddleware_httpauth.py | 3 +- tests/test_downloadermiddleware_httpcache.py | 3 +- ...st_downloadermiddleware_httpcompression.py | 31 ++++---- tests/test_downloadermiddleware_httpproxy.py | 3 +- tests/test_downloadermiddleware_offsite.py | 29 ++++---- tests/test_downloadermiddleware_redirect.py | 10 +-- ...wnloadermiddleware_redirect_metarefresh.py | 8 +- tests/test_downloadermiddleware_retry.py | 9 ++- tests/test_downloadermiddleware_robotstxt.py | 43 +++++++---- tests/test_downloadermiddleware_stats.py | 5 +- tests/test_downloadermiddleware_useragent.py | 3 +- tests/test_dupefilters.py | 9 ++- tests/test_engine.py | 3 +- tests/test_extension_debug.py | 11 +-- tests/test_extension_memdebug.py | 7 +- tests/test_extension_memusage.py | 3 +- tests/test_extension_periodic_log.py | 3 +- tests/test_extension_statsmailer.py | 7 +- tests/test_extension_telnet.py | 5 +- tests/test_extension_throttle.py | 18 ++--- tests/test_feedexport.py | 16 ++-- tests/test_feedexport_batch.py | 3 +- tests/test_feedexport_storages.py | 39 ++++++---- tests/test_feedexport_uri_params.py | 15 ++-- tests/test_logformatter.py | 16 ++-- tests/test_logstats.py | 7 +- tests/test_middleware.py | 3 +- tests/test_pipeline_files.py | 63 +++++++++------- tests/test_pipeline_images.py | 56 +++++++------- tests/test_pipeline_media.py | 9 ++- tests/test_pipelines.py | 3 +- tests/test_pqueues.py | 32 ++++---- tests/test_resolver.py | 5 +- tests/test_robotstxt_interface.py | 13 +++- tests/test_scheduler.py | 10 +-- tests/test_spidermiddleware.py | 11 +-- tests/test_spidermiddleware_base.py | 9 ++- tests/test_spidermiddleware_depth.py | 2 +- tests/test_spidermiddleware_httperror.py | 9 ++- tests/test_spidermiddleware_metacopy.py | 7 +- tests/test_spidermiddleware_urllength.py | 2 +- tests/test_spiderstate.py | 3 +- tests/test_squeues_request.py | 21 +++--- tests/test_stats.py | 5 +- tests/utils/bases/redirect.py | 74 +++++++++---------- 59 files changed, 405 insertions(+), 314 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index f9ee62838..2089b1224 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -28,6 +28,7 @@ from scrapy.utils.defer import ( maybe_deferred_to_future, ) from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: from collections.abc import Generator @@ -99,8 +100,8 @@ class Downloader: # AUTOTHROTTLE_START_DELAY. self._delay: float = self.settings.getfloat("DOWNLOAD_DELAY") self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY") - self.middleware: DownloaderMiddlewareManager = ( - DownloaderMiddlewareManager.from_crawler(crawler) + self.middleware: DownloaderMiddlewareManager = build_from_crawler( + DownloaderMiddlewareManager, crawler ) self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = None self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict( diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 6426b1751..351375d7b 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -36,7 +36,11 @@ from scrapy.utils.defer import ( ) from scrapy.utils.deprecate import method_is_overridden from scrapy.utils.log import failure_to_exc_info, logformatter_adapter -from scrapy.utils.misc import load_object, warn_on_generator_with_return_value +from scrapy.utils.misc import ( + build_from_crawler, + load_object, + warn_on_generator_with_return_value, +) from scrapy.utils.python import global_object_name from scrapy.utils.spider import iterate_spider_output @@ -102,13 +106,13 @@ class Slot: class Scraper: def __init__(self, crawler: Crawler) -> None: self.slot: Slot | None = None - self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler( - crawler + self.spidermw: SpiderMiddlewareManager = build_from_crawler( + SpiderMiddlewareManager, crawler ) itemproc_cls: type[ItemPipelineManager] = load_object( crawler.settings["ITEM_PROCESSOR"] ) - self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler) + self.itemproc: ItemPipelineManager = build_from_crawler(itemproc_cls, crawler) self._itemproc_has_async: dict[str, bool] = {} for method in [ "open_spider", diff --git a/scrapy/crawler.py b/scrapy/crawler.py index f0cfba6b9..444a5fb67 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -156,7 +156,7 @@ class Crawler: self.stats = load_object(self.settings["STATS_CLASS"])(self) lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"]) - self.logformatter = lf_cls.from_crawler(self) + self.logformatter = build_from_crawler(lf_cls, self) self.request_fingerprinter = build_from_crawler( load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]), @@ -200,7 +200,7 @@ class Crawler: logger.debug("Not using a Twisted reactor") self._apply_reactorless_default_settings() - self.extensions = ExtensionManager.from_crawler(self) + self.extensions = build_from_crawler(ExtensionManager, self) self.settings.freeze() d = dict(overridden_settings(self.settings)) diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index d7aa8738c..016cf9acb 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -18,7 +18,7 @@ from scrapy.http.request import NO_CALLBACK from scrapy.utils.decorators import _warn_spider_arg from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.misc import load_object +from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: # typing.Self requires Python 3.11 @@ -49,7 +49,7 @@ class RobotsTxtMiddleware: ) # check if parser dependencies are met, this should throw an error otherwise. - self._parserimpl.from_crawler(self.crawler, b"") + build_from_crawler(self._parserimpl, self.crawler, b"") @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -120,7 +120,7 @@ class RobotsTxtMiddleware: ) -> None: self._stats.inc_value("robotstxt/response_count") self._stats.inc_value(f"robotstxt/response_status_count/{response.status}") - rp = self._parserimpl.from_crawler(self.crawler, response.body) + rp = build_from_crawler(self._parserimpl, self.crawler, response.body) await self.crawler.signals.send_catch_log_async( signal=signals.robots_parsed, robotparser=rp, diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py index ec761cfbf..bf1f1bec4 100644 --- a/scrapy/extensions/memusage.py +++ b/scrapy/extensions/memusage.py @@ -19,6 +19,7 @@ from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call from scrapy.utils.defer import _schedule_coro from scrapy.utils.engine import get_engine_status +from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: from twisted.internet.task import LoopingCall @@ -57,7 +58,7 @@ class MemoryUsage: category=ScrapyDeprecationWarning, stacklevel=2, ) - self.mail = MailSender.from_crawler(crawler) + self.mail = build_from_crawler(MailSender, crawler) self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024 self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024 diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index 7647cf33d..3dc38dded 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.mail import MailSender +from scrapy.utils.misc import build_from_crawler if TYPE_CHECKING: from twisted.internet.defer import Deferred @@ -41,7 +42,7 @@ class StatsMailer: recipients: list[str] = crawler.settings.getlist("STATSMAILER_RCPTS") if not recipients: raise NotConfigured - mail: MailSender = MailSender.from_crawler(crawler) + mail: MailSender = build_from_crawler(MailSender, crawler) o = cls(crawler.stats, recipients, mail) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 6bc1ebbbb..44178727e 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -365,7 +365,7 @@ class TestShell: crawler.engine = MagicMock() crawler.engine.open_spider_async = AsyncMock() shell = Shell(crawler) - spider = Spider("test") + spider = Spider.from_crawler(crawler, "test") await shell._open_spider(spider) assert shell.spider is spider assert crawler.spider is spider diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py index 14de97b21..28a067fb7 100644 --- a/tests/test_downloader_handler_twisted_ftp.py +++ b/tests/test_downloader_handler_twisted_ftp.py @@ -212,4 +212,4 @@ class TestAnonymousFTP(TestFTPBase): def test_not_configured_without_reactor() -> None: crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) with pytest.raises(NotConfigured): - FTPDownloadHandler.from_crawler(crawler) + build_from_crawler(FTPDownloadHandler, crawler) diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index 32dfd7540..db353750d 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -11,6 +11,7 @@ from scrapy import Spider from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from scrapy.crawler import Crawler from scrapy.exceptions import NotConfigured +from scrapy.utils.misc import build_from_crawler from tests.utils.bases.download_handlers_http import ( TestHttpBase, TestHttpProxyBase, @@ -51,7 +52,7 @@ class HTTP11DownloadHandlerMixin: def test_not_configured_without_reactor() -> None: crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) with pytest.raises(NotConfigured): - HTTP11DownloadHandler.from_crawler(crawler) + build_from_crawler(HTTP11DownloadHandler, crawler) class TestHttp(HTTP11DownloadHandlerMixin, TestHttpBase): diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 9d4e161f3..2c3954b5e 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -13,6 +13,7 @@ from scrapy import Spider from scrapy.crawler import Crawler from scrapy.exceptions import DownloadFailedError, NotConfigured from scrapy.http import Request +from scrapy.utils.misc import build_from_crawler from tests.utils.bases.download_handlers_http import ( TestHttpProxyBase, TestHttpsBase, @@ -66,7 +67,7 @@ def test_not_configured_without_reactor() -> None: crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False}) with pytest.raises(NotConfigured): - H2DownloadHandler.from_crawler(crawler) + build_from_crawler(H2DownloadHandler, crawler) class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase): diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index 92eb18d33..9b89b0760 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -14,6 +14,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.utils.decorators import coroutine_test @@ -30,7 +31,7 @@ class TestManagerBase: async def get_mwman(self) -> AsyncGenerator[DownloaderMiddlewareManager]: crawler = get_crawler(Spider, self.settings_dict) crawler.spider = crawler._create_spider("foo") - mwman = DownloaderMiddlewareManager.from_crawler(crawler) + mwman = build_from_crawler(DownloaderMiddlewareManager, crawler) crawler.engine = crawler._create_engine() await crawler.engine.open_spider_async() try: diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index e4b66fe10..6c4c80eae 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -10,6 +10,7 @@ from scrapy.downloadermiddlewares.redirect import RedirectMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response from scrapy.http.request import CookiesT, VerboseCookie +from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.request import _to_verbose_cookies from scrapy.utils.spider import DefaultSpider @@ -72,8 +73,8 @@ class TestCookiesMiddleware: def setup_method(self): crawler = get_crawler(DefaultSpider) crawler.spider = crawler._create_spider() - self.mw = CookiesMiddleware.from_crawler(crawler) - self.redirect_middleware = RedirectMiddleware.from_crawler(crawler) + self.mw = build_from_crawler(CookiesMiddleware, crawler) + self.redirect_middleware = build_from_crawler(RedirectMiddleware, crawler) def teardown_method(self): del self.mw @@ -94,19 +95,19 @@ class TestCookiesMiddleware: def test_setting_false_cookies_enabled(self): with pytest.raises(NotConfigured): - CookiesMiddleware.from_crawler( - get_crawler(settings_dict={"COOKIES_ENABLED": False}) + build_from_crawler( + CookiesMiddleware, get_crawler(settings_dict={"COOKIES_ENABLED": False}) ) def test_setting_default_cookies_enabled(self): assert isinstance( - CookiesMiddleware.from_crawler(get_crawler()), CookiesMiddleware + build_from_crawler(CookiesMiddleware, get_crawler()), CookiesMiddleware ) def test_setting_true_cookies_enabled(self): assert isinstance( - CookiesMiddleware.from_crawler( - get_crawler(settings_dict={"COOKIES_ENABLED": True}) + build_from_crawler( + CookiesMiddleware, get_crawler(settings_dict={"COOKIES_ENABLED": True}) ), CookiesMiddleware, ) @@ -115,7 +116,7 @@ class TestCookiesMiddleware: self, caplog: pytest.LogCaptureFixture ) -> None: crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True}) - mw = CookiesMiddleware.from_crawler(crawler) + mw = build_from_crawler(CookiesMiddleware, crawler) caplog.clear() with caplog.at_level( logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies" @@ -145,7 +146,7 @@ class TestCookiesMiddleware: def test_debug_no_cookies(self, caplog: pytest.LogCaptureFixture) -> None: crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True}) - mw = CookiesMiddleware.from_crawler(crawler) + mw = build_from_crawler(CookiesMiddleware, crawler) caplog.clear() with caplog.at_level( logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies" @@ -161,7 +162,7 @@ class TestCookiesMiddleware: self, caplog: pytest.LogCaptureFixture ) -> None: crawler = get_crawler(settings_dict={"COOKIES_DEBUG": False}) - mw = CookiesMiddleware.from_crawler(crawler) + mw = build_from_crawler(CookiesMiddleware, crawler) caplog.clear() with caplog.at_level( logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies" diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py index 8c89c3ffb..507097df9 100644 --- a/tests/test_downloadermiddleware_defaultheaders.py +++ b/tests/test_downloadermiddleware_defaultheaders.py @@ -3,6 +3,7 @@ from __future__ import annotations from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler @@ -13,7 +14,7 @@ def get_defaults_mw() -> tuple[dict[bytes, list[bytes]], DefaultHeadersMiddlewar to_bytes(k): [to_bytes(v)] for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items() } - return defaults, DefaultHeadersMiddleware.from_crawler(crawler) + return defaults, build_from_crawler(DefaultHeadersMiddleware, crawler) def test_process_request(): diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py index 9b64cf349..7e2d77136 100644 --- a/tests/test_downloadermiddleware_downloadtimeout.py +++ b/tests/test_downloadermiddleware_downloadtimeout.py @@ -5,6 +5,7 @@ from typing import Any from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler @@ -12,7 +13,7 @@ def get_request_spider_mw(settings: dict[str, Any] | None = None): crawler = get_crawler(Spider, settings) spider = crawler._create_spider("foo") request = Request("http://scrapytest.org/") - return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler) + return request, spider, build_from_crawler(DownloadTimeoutMiddleware, crawler) def test_default_download_timeout(): diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py index dd5af3bc5..e4b414f45 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_httpauth.py @@ -7,6 +7,7 @@ from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler _DOMAIN_NOT_SET = object() @@ -21,7 +22,7 @@ def make_mw( } if domain is not _DOMAIN_NOT_SET: settings["HTTPAUTH_DOMAIN"] = domain - return HttpAuthMiddleware.from_crawler(get_crawler(settings_dict=settings)) + return build_from_crawler(HttpAuthMiddleware, get_crawler(settings_dict=settings)) # --- Spider attribute tests (deprecated) --- diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index dc8228470..50ff5e63f 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -17,6 +17,7 @@ from scrapy.exceptions import IgnoreRequest from scrapy.extensions.httpcache import DummyPolicy from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler if TYPE_CHECKING: @@ -87,7 +88,7 @@ class TestBase: def _middleware(self, **new_settings: Any) -> Generator[HttpCacheMiddleware]: with self._get_crawler(**new_settings) as crawler: assert crawler.spider - mw = HttpCacheMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCacheMiddleware, crawler) mw.spider_opened(crawler.spider) try: yield mw diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a43bb51ba..4e0fd7c8b 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -18,6 +18,7 @@ from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider from scrapy.utils._compression import _DecompressionMaxSizeExceeded from scrapy.utils.gz import gunzip +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler from tests import tests_datadir @@ -59,7 +60,7 @@ def _skip_if_no_zstd() -> None: class TestHttpCompression: def setup_method(self): self.crawler = get_crawler(Spider) - self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) + self.mw = build_from_crawler(HttpCompressionMiddleware, self.crawler) assert self.crawler.stats self.crawler.stats.open_spider() @@ -93,20 +94,22 @@ class TestHttpCompression: def test_setting_false_compression_enabled(self): with pytest.raises(NotConfigured): - HttpCompressionMiddleware.from_crawler( - get_crawler(settings_dict={"COMPRESSION_ENABLED": False}) + build_from_crawler( + HttpCompressionMiddleware, + get_crawler(settings_dict={"COMPRESSION_ENABLED": False}), ) def test_setting_default_compression_enabled(self): assert isinstance( - HttpCompressionMiddleware.from_crawler(get_crawler()), + build_from_crawler(HttpCompressionMiddleware, get_crawler()), HttpCompressionMiddleware, ) def test_setting_true_compression_enabled(self): assert isinstance( - HttpCompressionMiddleware.from_crawler( - get_crawler(settings_dict={"COMPRESSION_ENABLED": True}) + build_from_crawler( + HttpCompressionMiddleware, + get_crawler(settings_dict={"COMPRESSION_ENABLED": True}), ), HttpCompressionMiddleware, ) @@ -496,7 +499,7 @@ class TestHttpCompression: settings = {"DOWNLOAD_MAXSIZE": 1_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B @@ -525,7 +528,7 @@ class TestHttpCompression: settings = {"DOWNLOAD_MAXSIZE": 1_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse("bomb-gzip") # 11_511_612 B @@ -552,7 +555,7 @@ class TestHttpCompression: crawler = get_crawler(DownloadMaxSizeSpider) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") @@ -584,7 +587,7 @@ class TestHttpCompression: def _test_compression_bomb_request_meta(self, compression_id: str) -> None: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") @@ -616,7 +619,7 @@ class TestHttpCompression: settings = {"DOWNLOAD_WARNSIZE": 10_000_000} crawler = get_crawler(Spider, settings_dict=settings) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") @@ -668,7 +671,7 @@ class TestHttpCompression: crawler = get_crawler(DownloadWarnSizeSpider) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") @@ -721,7 +724,7 @@ class TestHttpCompression: ) -> None: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(f"bomb-{compression_id}") response.meta["download_warnsize"] = 10_000_000 @@ -769,7 +772,7 @@ class TestHttpCompression: def _get_truncated_response(self, compression_id: str) -> Response: crawler = get_crawler(Spider) spider = crawler._create_spider("scrapytest.org") - mw = HttpCompressionMiddleware.from_crawler(crawler) + mw = build_from_crawler(HttpCompressionMiddleware, crawler) mw.open_spider(spider) response = self._getresponse(compression_id) truncated_body = response.body[: len(response.body) // 2] diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 54d4601a7..a7a6f2079 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -6,6 +6,7 @@ from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware from scrapy.exceptions import NotConfigured from scrapy.http import Request from scrapy.spiders import Spider +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler @@ -20,7 +21,7 @@ class TestHttpProxyMiddleware: def test_not_enabled(self): crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False}) with pytest.raises(NotConfigured): - HttpProxyMiddleware.from_crawler(crawler) + build_from_crawler(HttpProxyMiddleware, crawler) def test_no_environment_proxies(self): os.environ.clear() diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index 1f91dcd4b..40e09be13 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -7,6 +7,7 @@ from scrapy import Request, Spider from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware from scrapy.exceptions import IgnoreRequest from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler UNSET = object() @@ -31,7 +32,7 @@ UNSET = object() def test_process_request_domain_filtering(allowed_domain, url, allowed): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request(url) if allowed: @@ -53,7 +54,7 @@ def test_process_request_domain_filtering(allowed_domain, url, allowed): def test_process_request_dont_filter(value, filtered): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) kwargs: dict[str, Any] = {} if value is not UNSET: @@ -82,7 +83,7 @@ def test_process_request_dont_filter(value, filtered): def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) kwargs: dict[str, Any] = {"meta": {}} if allow_offsite is not UNSET: @@ -111,7 +112,7 @@ def test_process_request_no_allowed_domains(value): if value is not UNSET: kwargs["allowed_domains"] = value crawler.spider = crawler._create_spider(name="a", **kwargs) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request("https://example.com") assert mw.process_request(request) is None @@ -121,7 +122,7 @@ def test_process_request_invalid_domains(): crawler = get_crawler(Spider) allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request("https://a.example") assert mw.process_request(request) is None @@ -150,7 +151,7 @@ def test_process_request_invalid_domains(): def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request(url) if allowed: @@ -172,7 +173,7 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed): def test_request_scheduled_dont_filter(value, filtered): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) kwargs: dict[str, Any] = {} if value is not UNSET: @@ -199,7 +200,7 @@ def test_request_scheduled_no_allowed_domains(value): if value is not UNSET: kwargs["allowed_domains"] = value crawler.spider = crawler._create_spider(name="a", **kwargs) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request("https://example.com") mw.request_scheduled(request, crawler.spider) @@ -209,7 +210,7 @@ def test_request_scheduled_invalid_domains(): crawler = get_crawler(Spider) allowed_domains = ["a.example", None, "http:////b.example", "//c.example"] crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request("https://a.example") mw.request_scheduled(request, crawler.spider) @@ -222,7 +223,7 @@ def test_request_scheduled_invalid_domains(): def test_repeated_offsite_domain(): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) req1 = Request("http://other.org/1") req2 = Request("http://other.org/2") @@ -246,7 +247,7 @@ def test_should_follow_override(): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"]) - mw = RootOnlyOffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(RootOnlyOffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) assert mw.process_request(Request("https://example.com/1")) is None with pytest.raises(IgnoreRequest): @@ -256,7 +257,7 @@ def test_should_follow_override(): def test_ignore_request_reason(): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"]) - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(crawler.spider) request = Request("http://other.org/1") with pytest.raises( @@ -274,7 +275,7 @@ def test_dynamic_allowed_domains(): crawler = get_crawler(DomainSpider) spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"]) crawler.spider = spider - mw = OffsiteMiddleware.from_crawler(crawler) + mw = build_from_crawler(OffsiteMiddleware, crawler) mw.spider_opened(spider) with pytest.raises(IgnoreRequest): @@ -300,7 +301,7 @@ def test_dynamic_allowed_domains_caching(): crawler = get_crawler(DomainSpider) spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"]) crawler.spider = spider - mw = TrackingMiddleware.from_crawler(crawler) + mw = build_from_crawler(TrackingMiddleware, crawler) mw.spider_opened(spider) for _ in range(3): diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index de97aaadb..4cebcb281 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -27,7 +27,7 @@ class TestRedirectMiddleware(TestRedirectBase): def setup_method(self): crawler = get_crawler(DefaultSpider) crawler.spider = crawler._create_spider() - self.mw = self.mwcls.from_crawler(crawler) + self.mw = build_from_crawler(self.mwcls, crawler) def get_response(self, request, location, status=302): headers = {"Location": location} @@ -208,7 +208,7 @@ class TestRedirectMiddleware(TestRedirectBase): response = Response(source_url, headers=resp_headers, status=302) crawler = get_crawler() referer_mw = build_from_crawler(RefererMiddleware, crawler) - redirect_mw = self.mwcls.from_crawler(crawler) + redirect_mw = build_from_crawler(self.mwcls, crawler) redirect_mw._referer_spider_middleware = referer_mw redirect_request = redirect_mw.process_response(source_request, response) if expected_referer: @@ -223,7 +223,7 @@ class TestRedirectMiddleware(TestRedirectBase): source_url, headers={"Referer": "http://example.com/old"} ) response = Response(source_url, headers={"Location": redirect_url}, status=302) - redirect_mw = self.mwcls.from_crawler(get_crawler()) + redirect_mw = build_from_crawler(self.mwcls, get_crawler()) redirect_mw._referer_spider_middleware = None redirect_request = redirect_mw.process_response(source_request, response) assert "Referer" not in redirect_request.headers @@ -352,7 +352,7 @@ class TestRedirectMiddleware(TestRedirectBase): @pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES) def test_redirect_schemes(url, location, target): crawler = get_crawler(Spider) - mw = RedirectMiddleware.from_crawler(crawler) + mw = build_from_crawler(RedirectMiddleware, crawler) request = Request(url) response = Response(url, headers={"Location": location}, status=301) redirect = mw.process_response(request, response) @@ -477,4 +477,4 @@ def test_warning_subclass(caplog): def test_not_configured(): crawler = get_crawler(DefaultSpider, {"REDIRECT_ENABLED": False}) with pytest.raises(NotConfigured): - RedirectMiddleware.from_crawler(crawler) + build_from_crawler(RedirectMiddleware, crawler) diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py index 83dc6825f..3887ddd26 100644 --- a/tests/test_downloadermiddleware_redirect_metarefresh.py +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -32,7 +32,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase): def setup_method(self): crawler = get_crawler(Spider) - self.mw = self.mwcls.from_crawler(crawler) + self.mw = build_from_crawler(self.mwcls, crawler) def _body( self, interval: int = 5, url: str = "http://example.org/newpage" @@ -95,7 +95,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase): """Test that Scrapy 1.x behavior remains possible""" settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]} crawler = get_crawler(Spider, settings) - mw = MetaRefreshMiddleware.from_crawler(crawler) + mw = build_from_crawler(MetaRefreshMiddleware, crawler) req = Request(url="http://example.org") body = ( """