RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into faster-shutdown-2

This commit is contained in:
Adrian Chaves 2026-08-10 19:29:29 +02:00
parent 363d720298 95c16aa9af
commit 75a72fd6b0
61 changed files with 443 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/core/downloader/__init__.py
View File

@ -28,6 +28,7 @@ from scrapy.utils.defer import (
maybe_deferred_to_future,
)
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
from collections.abc import Generator
@ -99,8 +100,8 @@ class Downloader:
# AUTOTHROTTLE_START_DELAY.
self._delay: float = self.settings.getfloat("DOWNLOAD_DELAY")
self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")
self.middleware: DownloaderMiddlewareManager = (
DownloaderMiddlewareManager.from_crawler(crawler)
self.middleware: DownloaderMiddlewareManager = build_from_crawler(
DownloaderMiddlewareManager, crawler
)
self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = None
self._accepting_requests: bool = True

12
scrapy/core/scraper.py
View File

@ -36,7 +36,11 @@ from scrapy.utils.defer import (
)
from scrapy.utils.deprecate import method_is_overridden
from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
from scrapy.utils.misc import (
build_from_crawler,
load_object,
warn_on_generator_with_return_value,
)
from scrapy.utils.python import global_object_name
from scrapy.utils.spider import iterate_spider_output
@ -102,13 +106,13 @@ class Slot:
class Scraper:
def __init__(self, crawler: Crawler) -> None:
self.slot: Slot | None = None
self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler(
crawler
self.spidermw: SpiderMiddlewareManager = build_from_crawler(
SpiderMiddlewareManager, crawler
)
itemproc_cls: type[ItemPipelineManager] = load_object(
crawler.settings["ITEM_PROCESSOR"]
)
self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler)
self.itemproc: ItemPipelineManager = build_from_crawler(itemproc_cls, crawler)
self._itemproc_has_async: dict[str, bool] = {}
for method in [
"open_spider",

4
scrapy/crawler.py
View File

@ -166,7 +166,7 @@ class Crawler:
self.stats = load_object(self.settings["STATS_CLASS"])(self)
lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
self.logformatter = lf_cls.from_crawler(self)
self.logformatter = build_from_crawler(lf_cls, self)
self.request_fingerprinter = build_from_crawler(
load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
@ -210,7 +210,7 @@ class Crawler:
logger.debug("Not using a Twisted reactor")
self._apply_reactorless_default_settings()
self.extensions = ExtensionManager.from_crawler(self)
self.extensions = build_from_crawler(ExtensionManager, self)
self.settings.freeze()
d = dict(overridden_settings(self.settings))

6
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -18,7 +18,7 @@ from scrapy.http.request import NO_CALLBACK
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import load_object
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
# typing.Self requires Python 3.11
@ -49,7 +49,7 @@ class RobotsTxtMiddleware:
)
# check if parser dependencies are met, this should throw an error otherwise.
self._parserimpl.from_crawler(self.crawler, b"")
build_from_crawler(self._parserimpl, self.crawler, b"")
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:
@ -120,7 +120,7 @@ class RobotsTxtMiddleware:
) -> None:
self._stats.inc_value("robotstxt/response_count")
self._stats.inc_value(f"robotstxt/response_status_count/{response.status}")
rp = self._parserimpl.from_crawler(self.crawler, response.body)
rp = build_from_crawler(self._parserimpl, self.crawler, response.body)
await self.crawler.signals.send_catch_log_async(
signal=signals.robots_parsed,
robotparser=rp,

3
scrapy/extensions/memusage.py
View File

@ -19,6 +19,7 @@ from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
from scrapy.utils.defer import _schedule_coro
from scrapy.utils.engine import get_engine_status
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
from twisted.internet.task import LoopingCall
@ -57,7 +58,7 @@ class MemoryUsage:
category=ScrapyDeprecationWarning,
stacklevel=2,
)
self.mail = MailSender.from_crawler(crawler)
self.mail = build_from_crawler(MailSender, crawler)
self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024
self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024

3
scrapy/extensions/statsmailer.py
View File

@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.mail import MailSender
from scrapy.utils.misc import build_from_crawler
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
@ -41,7 +42,7 @@ class StatsMailer:
recipients: list[str] = crawler.settings.getlist("STATSMAILER_RCPTS")
if not recipients:
raise NotConfigured
mail: MailSender = MailSender.from_crawler(crawler)
mail: MailSender = build_from_crawler(MailSender, crawler)
o = cls(crawler.stats, recipients, mail)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o

20
scrapy/utils/log.py
View File

@ -2,7 +2,9 @@ from __future__ import annotations
import logging
import pprint
import re
import sys
import warnings
from collections.abc import MutableMapping
from logging.config import dictConfig
from typing import TYPE_CHECKING, Any, cast
@ -12,6 +14,7 @@ from twisted.python import log as twisted_log
from twisted.python.failure import Failure
import scrapy
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
from scrapy.utils.versions import get_versions
@ -242,6 +245,9 @@ class LogCounterHandler(logging.Handler):
self.crawler.stats.inc_value(sname)
_MSG_MAPPING_PLACEHOLDER = re.compile(r"%\(\w+\)")
def logformatter_adapter(
logkws: LogFormatterResult,
) -> tuple[Any, ...]:
@ -257,6 +263,20 @@ def logformatter_adapter(
# argument, so empty args are left out. Tuple args become one positional
# argument each, while a dict is a single positional argument.
if not args:
if _MSG_MAPPING_PLACEHOLDER.search(message):
# The log formatter method has already returned, so there is no
# frame of it left in the stack to point at. msg is part of the
# warning message instead, so that each offending method gets its
# own warning.
warnings.warn(
f"A log formatter method returned msg {message!r} with "
f"%(name)s placeholders and no args. Interpolating msg with "
f"the returned dict is deprecated, return those values under "
f"args instead.",
ScrapyDeprecationWarning,
stacklevel=1,
)
return (level, message, logkws)
return (level, message)
if isinstance(args, tuple):
return (level, message, *args)

2
tests/test_command_shell.py
View File

@ -365,7 +365,7 @@ class TestShell:
crawler.engine = MagicMock()
crawler.engine.open_spider_async = AsyncMock()
shell = Shell(crawler)
spider = Spider("test")
spider = Spider.from_crawler(crawler, "test")
await shell._open_spider(spider)
assert shell.spider is spider
assert crawler.spider is spider

2
tests/test_downloader_handler_twisted_ftp.py
View File

@ -212,4 +212,4 @@ class TestAnonymousFTP(TestFTPBase):
def test_not_configured_without_reactor() -> None:
crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False})
with pytest.raises(NotConfigured):
FTPDownloadHandler.from_crawler(crawler)
build_from_crawler(FTPDownloadHandler, crawler)

3
tests/test_downloader_handler_twisted_http11.py
View File

@ -11,6 +11,7 @@ from scrapy import Spider
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
from scrapy.crawler import Crawler
from scrapy.exceptions import NotConfigured
from scrapy.utils.misc import build_from_crawler
from tests.utils.bases.download_handlers_http import (
TestHttpBase,
TestHttpProxyBase,
@ -51,7 +52,7 @@ class HTTP11DownloadHandlerMixin:
def test_not_configured_without_reactor() -> None:
crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False})
with pytest.raises(NotConfigured):
HTTP11DownloadHandler.from_crawler(crawler)
build_from_crawler(HTTP11DownloadHandler, crawler)
class TestHttp(HTTP11DownloadHandlerMixin, TestHttpBase):

3
tests/test_downloader_handler_twisted_http2.py
View File

@ -13,6 +13,7 @@ from scrapy import Spider
from scrapy.crawler import Crawler
from scrapy.exceptions import DownloadFailedError, NotConfigured
from scrapy.http import Request
from scrapy.utils.misc import build_from_crawler
from tests.utils.bases.download_handlers_http import (
TestHttpProxyBase,
TestHttpsBase,
@ -66,7 +67,7 @@ def test_not_configured_without_reactor() -> None:
crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False})
with pytest.raises(NotConfigured):
H2DownloadHandler.from_crawler(crawler)
build_from_crawler(H2DownloadHandler, crawler)
class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):

3
tests/test_downloadermiddleware.py
View File

@ -14,6 +14,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.utils.decorators import coroutine_test
@ -30,7 +31,7 @@ class TestManagerBase:
async def get_mwman(self) -> AsyncGenerator[DownloaderMiddlewareManager]:
crawler = get_crawler(Spider, self.settings_dict)
crawler.spider = crawler._create_spider("foo")
mwman = DownloaderMiddlewareManager.from_crawler(crawler)
mwman = build_from_crawler(DownloaderMiddlewareManager, crawler)
crawler.engine = crawler._create_engine()
await crawler.engine.open_spider_async()
try:

21
tests/test_downloadermiddleware_cookies.py
View File

@ -10,6 +10,7 @@ from scrapy.downloadermiddlewares.redirect import RedirectMiddleware
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import CookiesT, VerboseCookie
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.request import _to_verbose_cookies
from scrapy.utils.spider import DefaultSpider
@ -72,8 +73,8 @@ class TestCookiesMiddleware:
def setup_method(self):
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
self.mw = CookiesMiddleware.from_crawler(crawler)
self.redirect_middleware = RedirectMiddleware.from_crawler(crawler)
self.mw = build_from_crawler(CookiesMiddleware, crawler)
self.redirect_middleware = build_from_crawler(RedirectMiddleware, crawler)
def teardown_method(self):
del self.mw
@ -94,19 +95,19 @@ class TestCookiesMiddleware:
def test_setting_false_cookies_enabled(self):
with pytest.raises(NotConfigured):
CookiesMiddleware.from_crawler(
get_crawler(settings_dict={"COOKIES_ENABLED": False})
build_from_crawler(
CookiesMiddleware, get_crawler(settings_dict={"COOKIES_ENABLED": False})
)
def test_setting_default_cookies_enabled(self):
assert isinstance(
CookiesMiddleware.from_crawler(get_crawler()), CookiesMiddleware
build_from_crawler(CookiesMiddleware, get_crawler()), CookiesMiddleware
)
def test_setting_true_cookies_enabled(self):
assert isinstance(
CookiesMiddleware.from_crawler(
get_crawler(settings_dict={"COOKIES_ENABLED": True})
build_from_crawler(
CookiesMiddleware, get_crawler(settings_dict={"COOKIES_ENABLED": True})
),
CookiesMiddleware,
)
@ -115,7 +116,7 @@ class TestCookiesMiddleware:
self, caplog: pytest.LogCaptureFixture
) -> None:
crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True})
mw = CookiesMiddleware.from_crawler(crawler)
mw = build_from_crawler(CookiesMiddleware, crawler)
caplog.clear()
with caplog.at_level(
logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies"
@ -145,7 +146,7 @@ class TestCookiesMiddleware:
def test_debug_no_cookies(self, caplog: pytest.LogCaptureFixture) -> None:
crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True})
mw = CookiesMiddleware.from_crawler(crawler)
mw = build_from_crawler(CookiesMiddleware, crawler)
caplog.clear()
with caplog.at_level(
logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies"
@ -161,7 +162,7 @@ class TestCookiesMiddleware:
self, caplog: pytest.LogCaptureFixture
) -> None:
crawler = get_crawler(settings_dict={"COOKIES_DEBUG": False})
mw = CookiesMiddleware.from_crawler(crawler)
mw = build_from_crawler(CookiesMiddleware, crawler)
caplog.clear()
with caplog.at_level(
logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies"

3
tests/test_downloadermiddleware_defaultheaders.py
View File

@ -3,6 +3,7 @@ from __future__ import annotations
from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler
@ -13,7 +14,7 @@ def get_defaults_mw() -> tuple[dict[bytes, list[bytes]], DefaultHeadersMiddlewar
to_bytes(k): [to_bytes(v)]
for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()
}
return defaults, DefaultHeadersMiddleware.from_crawler(crawler)
return defaults, build_from_crawler(DefaultHeadersMiddleware, crawler)
def test_process_request():

3
tests/test_downloadermiddleware_downloadtimeout.py
View File

@ -5,6 +5,7 @@ from typing import Any
from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
@ -12,7 +13,7 @@ def get_request_spider_mw(settings: dict[str, Any] | None = None):
crawler = get_crawler(Spider, settings)
spider = crawler._create_spider("foo")
request = Request("http://scrapytest.org/")
return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
return request, spider, build_from_crawler(DownloadTimeoutMiddleware, crawler)
def test_default_download_timeout():

3
tests/test_downloadermiddleware_httpauth.py
View File

@ -7,6 +7,7 @@ from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
_DOMAIN_NOT_SET = object()
@ -21,7 +22,7 @@ def make_mw(
}
if domain is not _DOMAIN_NOT_SET:
settings["HTTPAUTH_DOMAIN"] = domain
return HttpAuthMiddleware.from_crawler(get_crawler(settings_dict=settings))
return build_from_crawler(HttpAuthMiddleware, get_crawler(settings_dict=settings))
# --- Spider attribute tests (deprecated) ---

3
tests/test_downloadermiddleware_httpcache.py
View File

@ -17,6 +17,7 @@ from scrapy.exceptions import IgnoreRequest
from scrapy.extensions.httpcache import DummyPolicy
from scrapy.http import HtmlResponse, Request, Response
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@ -87,7 +88,7 @@ class TestBase:
def _middleware(self, **new_settings: Any) -> Generator[HttpCacheMiddleware]:
with self._get_crawler(**new_settings) as crawler:
assert crawler.spider
mw = HttpCacheMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCacheMiddleware, crawler)
mw.spider_opened(crawler.spider)
try:
yield mw

31
tests/test_downloadermiddleware_httpcompression.py
View File

@ -18,6 +18,7 @@ from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils._compression import _DecompressionMaxSizeExceeded
from scrapy.utils.gz import gunzip
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests import tests_datadir
@ -59,7 +60,7 @@ def _skip_if_no_zstd() -> None:
class TestHttpCompression:
def setup_method(self):
self.crawler = get_crawler(Spider)
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
self.mw = build_from_crawler(HttpCompressionMiddleware, self.crawler)
assert self.crawler.stats
self.crawler.stats.open_spider()
@ -93,20 +94,22 @@ class TestHttpCompression:
def test_setting_false_compression_enabled(self):
with pytest.raises(NotConfigured):
HttpCompressionMiddleware.from_crawler(
get_crawler(settings_dict={"COMPRESSION_ENABLED": False})
build_from_crawler(
HttpCompressionMiddleware,
get_crawler(settings_dict={"COMPRESSION_ENABLED": False}),
)
def test_setting_default_compression_enabled(self):
assert isinstance(
HttpCompressionMiddleware.from_crawler(get_crawler()),
build_from_crawler(HttpCompressionMiddleware, get_crawler()),
HttpCompressionMiddleware,
)
def test_setting_true_compression_enabled(self):
assert isinstance(
HttpCompressionMiddleware.from_crawler(
get_crawler(settings_dict={"COMPRESSION_ENABLED": True})
build_from_crawler(
HttpCompressionMiddleware,
get_crawler(settings_dict={"COMPRESSION_ENABLED": True}),
),
HttpCompressionMiddleware,
)
@ -496,7 +499,7 @@ class TestHttpCompression:
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B
@ -525,7 +528,7 @@ class TestHttpCompression:
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse("bomb-gzip") # 11_511_612 B
@ -552,7 +555,7 @@ class TestHttpCompression:
crawler = get_crawler(DownloadMaxSizeSpider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
@ -584,7 +587,7 @@ class TestHttpCompression:
def _test_compression_bomb_request_meta(self, compression_id: str) -> None:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
@ -616,7 +619,7 @@ class TestHttpCompression:
settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
@ -668,7 +671,7 @@ class TestHttpCompression:
crawler = get_crawler(DownloadWarnSizeSpider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
@ -721,7 +724,7 @@ class TestHttpCompression:
) -> None:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_warnsize"] = 10_000_000
@ -769,7 +772,7 @@ class TestHttpCompression:
def _get_truncated_response(self, compression_id: str) -> Response:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpCompressionMiddleware, crawler)
mw.open_spider(spider)
response = self._getresponse(compression_id)
truncated_body = response.body[: len(response.body) // 2]

3
tests/test_downloadermiddleware_httpproxy.py
View File

@ -6,6 +6,7 @@ from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
from scrapy.exceptions import NotConfigured
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
@ -20,7 +21,7 @@ class TestHttpProxyMiddleware:
def test_not_enabled(self):
crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False})
with pytest.raises(NotConfigured):
HttpProxyMiddleware.from_crawler(crawler)
build_from_crawler(HttpProxyMiddleware, crawler)
def test_no_environment_proxies(self):
os.environ.clear()

29
tests/test_downloadermiddleware_offsite.py
View File

@ -7,6 +7,7 @@ from scrapy import Request, Spider
from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
UNSET = object()
@ -31,7 +32,7 @@ UNSET = object()
def test_process_request_domain_filtering(allowed_domain, url, allowed):
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request(url)
if allowed:
@ -53,7 +54,7 @@ def test_process_request_domain_filtering(allowed_domain, url, allowed):
def test_process_request_dont_filter(value, filtered):
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
kwargs: dict[str, Any] = {}
if value is not UNSET:
@ -82,7 +83,7 @@ def test_process_request_dont_filter(value, filtered):
def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
kwargs: dict[str, Any] = {"meta": {}}
if allow_offsite is not UNSET:
@ -111,7 +112,7 @@ def test_process_request_no_allowed_domains(value):
if value is not UNSET:
kwargs["allowed_domains"] = value
crawler.spider = crawler._create_spider(name="a", **kwargs)
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request("https://example.com")
assert mw.process_request(request) is None
@ -121,7 +122,7 @@ def test_process_request_invalid_domains():
crawler = get_crawler(Spider)
allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request("https://a.example")
assert mw.process_request(request) is None
@ -150,7 +151,7 @@ def test_process_request_invalid_domains():
def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=[allowed_domain])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request(url)
if allowed:
@ -172,7 +173,7 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
def test_request_scheduled_dont_filter(value, filtered):
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
kwargs: dict[str, Any] = {}
if value is not UNSET:
@ -199,7 +200,7 @@ def test_request_scheduled_no_allowed_domains(value):
if value is not UNSET:
kwargs["allowed_domains"] = value
crawler.spider = crawler._create_spider(name="a", **kwargs)
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request("https://example.com")
mw.request_scheduled(request, crawler.spider)
@ -209,7 +210,7 @@ def test_request_scheduled_invalid_domains():
crawler = get_crawler(Spider)
allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request("https://a.example")
mw.request_scheduled(request, crawler.spider)
@ -222,7 +223,7 @@ def test_request_scheduled_invalid_domains():
def test_repeated_offsite_domain():
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
req1 = Request("http://other.org/1")
req2 = Request("http://other.org/2")
@ -246,7 +247,7 @@ def test_should_follow_override():
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
mw = RootOnlyOffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(RootOnlyOffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
assert mw.process_request(Request("https://example.com/1")) is None
with pytest.raises(IgnoreRequest):
@ -256,7 +257,7 @@ def test_should_follow_override():
def test_ignore_request_reason():
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(crawler.spider)
request = Request("http://other.org/1")
with pytest.raises(
@ -274,7 +275,7 @@ def test_dynamic_allowed_domains():
crawler = get_crawler(DomainSpider)
spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"])
crawler.spider = spider
mw = OffsiteMiddleware.from_crawler(crawler)
mw = build_from_crawler(OffsiteMiddleware, crawler)
mw.spider_opened(spider)
with pytest.raises(IgnoreRequest):
@ -300,7 +301,7 @@ def test_dynamic_allowed_domains_caching():
crawler = get_crawler(DomainSpider)
spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"])
crawler.spider = spider
mw = TrackingMiddleware.from_crawler(crawler)
mw = build_from_crawler(TrackingMiddleware, crawler)
mw.spider_opened(spider)
for _ in range(3):

10
tests/test_downloadermiddleware_redirect.py
View File

@ -27,7 +27,7 @@ class TestRedirectMiddleware(TestRedirectBase):
def setup_method(self):
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
self.mw = self.mwcls.from_crawler(crawler)
self.mw = build_from_crawler(self.mwcls, crawler)
def get_response(self, request, location, status=302):
headers = {"Location": location}
@ -208,7 +208,7 @@ class TestRedirectMiddleware(TestRedirectBase):
response = Response(source_url, headers=resp_headers, status=302)
crawler = get_crawler()
referer_mw = build_from_crawler(RefererMiddleware, crawler)
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
redirect_mw._referer_spider_middleware = referer_mw
redirect_request = redirect_mw.process_response(source_request, response)
if expected_referer:
@ -223,7 +223,7 @@ class TestRedirectMiddleware(TestRedirectBase):
source_url, headers={"Referer": "http://example.com/old"}
)
response = Response(source_url, headers={"Location": redirect_url}, status=302)
redirect_mw = self.mwcls.from_crawler(get_crawler())
redirect_mw = build_from_crawler(self.mwcls, get_crawler())
redirect_mw._referer_spider_middleware = None
redirect_request = redirect_mw.process_response(source_request, response)
assert "Referer" not in redirect_request.headers
@ -352,7 +352,7 @@ class TestRedirectMiddleware(TestRedirectBase):
@pytest.mark.parametrize(SCHEME_PARAMS, REDIRECT_SCHEME_CASES)
def test_redirect_schemes(url, location, target):
crawler = get_crawler(Spider)
mw = RedirectMiddleware.from_crawler(crawler)
mw = build_from_crawler(RedirectMiddleware, crawler)
request = Request(url)
response = Response(url, headers={"Location": location}, status=301)
redirect = mw.process_response(request, response)
@ -477,4 +477,4 @@ def test_warning_subclass(caplog):
def test_not_configured():
crawler = get_crawler(DefaultSpider, {"REDIRECT_ENABLED": False})
with pytest.raises(NotConfigured):
RedirectMiddleware.from_crawler(crawler)
build_from_crawler(RedirectMiddleware, crawler)

8
tests/test_downloadermiddleware_redirect_metarefresh.py
View File

@ -32,7 +32,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
def setup_method(self):
crawler = get_crawler(Spider)
self.mw = self.mwcls.from_crawler(crawler)
self.mw = build_from_crawler(self.mwcls, crawler)
def _body(
self, interval: int = 5, url: str = "http://example.org/newpage"
@ -95,7 +95,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
"""Test that Scrapy 1.x behavior remains possible"""
settings = {"METAREFRESH_IGNORE_TAGS": ["script", "noscript"]}
crawler = get_crawler(Spider, settings)
mw = MetaRefreshMiddleware.from_crawler(crawler)
mw = build_from_crawler(MetaRefreshMiddleware, crawler)
req = Request(url="http://example.org")
body = (
"""<noscript><meta http-equiv="refresh" """
@ -134,7 +134,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
)
def test_meta_refresh_schemes(url, location, target):
crawler = get_crawler(Spider)
mw = MetaRefreshMiddleware.from_crawler(crawler)
mw = build_from_crawler(MetaRefreshMiddleware, crawler)
request = Request(url)
response = HtmlResponse(url, body=meta_refresh_body(location))
redirect = mw.process_response(request, response)
@ -169,4 +169,4 @@ def test_warning_meta_refresh_middleware(caplog):
def test_not_configured():
crawler = get_crawler(Spider, {"METAREFRESH_ENABLED": False})
with pytest.raises(NotConfigured):
MetaRefreshMiddleware.from_crawler(crawler)
build_from_crawler(MetaRefreshMiddleware, crawler)

9
tests/test_downloadermiddleware_retry.py
View File

@ -16,6 +16,7 @@ from scrapy.exceptions import (
from scrapy.http import Request, Response
from scrapy.settings.default_settings import RETRY_EXCEPTIONS
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
@ -24,7 +25,7 @@ class TestRetry:
def setup_method(self):
self.crawler = get_crawler(DefaultSpider)
self.crawler.spider = self.crawler._create_spider()
self.mw = RetryMiddleware.from_crawler(self.crawler)
self.mw = build_from_crawler(RetryMiddleware, self.crawler)
self.mw.max_retry_times = 2
def test_priority_adjust(self):
@ -94,7 +95,7 @@ class TestRetry:
DefaultSpider, settings_dict={"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"}
)
crawler.spider = crawler._create_spider()
mw = RetryMiddleware.from_crawler(crawler)
mw = build_from_crawler(RetryMiddleware, crawler)
mw.max_retry_times = 0
req = Request("http://example.com/503")
rsp = Response("http://example.com/503", body=b"", status=503)
@ -148,7 +149,7 @@ class TestRetry:
}
crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
crawler.spider = crawler._create_spider()
mw = RetryMiddleware.from_crawler(crawler)
mw = build_from_crawler(RetryMiddleware, crawler)
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
self._test_retry_exception(req, exc("foo"), mw)
@ -178,7 +179,7 @@ class TestMaxRetryTimes:
def get_middleware(self, settings: dict[str, Any] | None = None) -> RetryMiddleware:
crawler = get_crawler(DefaultSpider, settings or {})
crawler.spider = crawler._create_spider()
return RetryMiddleware.from_crawler(crawler)
return build_from_crawler(RetryMiddleware, crawler)
def test_with_settings_zero(self):
max_retry_times = 0

43
tests/test_downloadermiddleware_robotstxt.py
View File

@ -15,6 +15,7 @@ from scrapy.http.request import NO_CALLBACK
from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from tests.utils.decorators import coroutine_test
from tests.utils.robotstxt import rerp_available
@ -33,7 +34,7 @@ class TestRobotsTxtMiddleware:
self.crawler.settings = Settings()
self.crawler.settings.set("USER_AGENT", "CustomAgent")
with pytest.raises(NotConfigured):
RobotsTxtMiddleware(self.crawler)
build_from_crawler(RobotsTxtMiddleware, self.crawler)
def _get_successful_crawler(self) -> mock.MagicMock:
crawler = self.crawler
@ -60,7 +61,9 @@ Disallow: /some/randome/page.html
@coroutine_test
async def test_robotstxt(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_successful_crawler()
)
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
self.assertRobotsTxtRequested("http://site.local")
await self.assertIgnored(Request("http://site.local/admin/main"), middleware)
@ -89,7 +92,9 @@ Disallow: /some/randome/page.html
@coroutine_test
async def test_robotstxt_multiple_reqs(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_successful_crawler()
)
d1 = deferred_from_coro(
middleware.process_request(Request("http://site.local/allowed1"))
)
@ -101,20 +106,26 @@ Disallow: /some/randome/page.html
@pytest.mark.only_asyncio
@coroutine_test
async def test_robotstxt_multiple_reqs_asyncio(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_successful_crawler()
)
c1 = middleware.process_request(Request("http://site.local/allowed1"))
c2 = middleware.process_request(Request("http://site.local/allowed2"))
await asyncio.gather(c1, c2)
@coroutine_test
async def test_robotstxt_ready_parser(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_successful_crawler()
)
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
@coroutine_test
async def test_robotstxt_meta(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_successful_crawler()
)
meta = {"dont_obey_robotstxt": True}
await self.assertNotIgnored(
Request("http://site.local/allowed", meta=meta), middleware
@ -144,7 +155,9 @@ Disallow: /some/randome/page.html
@coroutine_test
async def test_robotstxt_garbage(self):
# garbage response should be discarded, equal 'allow all'
middleware = RobotsTxtMiddleware(self._get_garbage_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_garbage_crawler()
)
await self.assertNotIgnored(Request("http://site.local"), middleware)
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)
@ -166,7 +179,9 @@ Disallow: /some/randome/page.html
@coroutine_test
async def test_robotstxt_empty_response(self):
# empty response should equal 'allow all'
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_emptybody_crawler()
)
await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)
await self.assertNotIgnored(Request("http://site.local/static/"), middleware)
@ -183,7 +198,7 @@ Disallow: /some/randome/page.html
self.crawler.engine.download_async.side_effect = return_failure
middleware = RobotsTxtMiddleware(self.crawler)
middleware = build_from_crawler(RobotsTxtMiddleware, self.crawler)
await middleware.process_request(Request("http://site.local"))
assert "Robotstxt address not found" in caplog.text
@ -197,7 +212,7 @@ Disallow: /some/randome/page.html
self.crawler.engine.download_async.side_effect = immediate_failure
middleware = RobotsTxtMiddleware(self.crawler)
middleware = build_from_crawler(RobotsTxtMiddleware, self.crawler)
await self.assertNotIgnored(Request("http://site.local"), middleware)
@coroutine_test
@ -211,7 +226,7 @@ Disallow: /some/randome/page.html
self.crawler.engine.download_async.side_effect = ignore_request
middleware = RobotsTxtMiddleware(self.crawler)
middleware = build_from_crawler(RobotsTxtMiddleware, self.crawler)
with mock.patch(
"scrapy.downloadermiddlewares.robotstxt.logger"
) as mw_module_logger:
@ -224,14 +239,16 @@ Disallow: /some/randome/page.html
crawler = self._get_successful_crawler()
crawler.settings.set("ROBOTSTXT_USER_AGENT", "Examplebot")
crawler.settings.set("USER_AGENT", "Mozilla/5.0 (X11; Linux x86_64)")
middleware = RobotsTxtMiddleware(crawler)
middleware = build_from_crawler(RobotsTxtMiddleware, crawler)
rp = mock.MagicMock(return_value=True)
middleware.process_request_2(rp, Request("http://site.local/allowed"))
rp.allowed.assert_called_once_with("http://site.local/allowed", "Examplebot")
@coroutine_test
async def test_robotstxt_local_file(self):
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
middleware = build_from_crawler(
RobotsTxtMiddleware, self._get_emptybody_crawler()
)
middleware.process_request_2 = mock.MagicMock() # type: ignore[method-assign]
await middleware.process_request(Request("data:text/plain,Hello World data"))

5
tests/test_downloadermiddleware_stats.py
View File

@ -6,6 +6,7 @@ from scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_size
from scrapy.exceptions import NotConfigured
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
@ -17,7 +18,7 @@ class TestDownloaderStats:
def setup_method(self) -> None:
self.crawler = get_crawler(Spider)
assert self.crawler.stats is not None
self.mw = DownloaderStats(self.crawler.stats)
self.mw = build_from_crawler(DownloaderStats, self.crawler)
self.crawler.stats.open_spider()
@ -49,7 +50,7 @@ class TestDownloaderStats:
def test_from_crawler_not_configured(self) -> None:
crawler = get_crawler(Spider, {"DOWNLOADER_STATS": False})
with pytest.raises(NotConfigured):
DownloaderStats.from_crawler(crawler)
build_from_crawler(DownloaderStats, crawler)
def teardown_method(self) -> None:
assert self.crawler.stats is not None

3
tests/test_downloadermiddleware_useragent.py
View File

@ -3,6 +3,7 @@ from __future__ import annotations
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
@ -11,7 +12,7 @@ def get_spider_and_mw(
) -> tuple[Spider, UserAgentMiddleware]:
crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})
spider = crawler._create_spider("foo")
return spider, UserAgentMiddleware.from_crawler(crawler)
return spider, build_from_crawler(UserAgentMiddleware, crawler)
def test_default_agent():

9
tests/test_dupefilters.py
View File

@ -14,6 +14,7 @@ from scrapy.core.scheduler import Scheduler
from scrapy.dupefilters import BaseDupeFilter, RFPDupeFilter
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
@ -30,7 +31,7 @@ def _get_dupefilter(
) -> BaseDupeFilter:
if crawler is None:
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
scheduler = build_from_crawler(Scheduler, crawler)
dupefilter = scheduler.df
if open_:
dupefilter.open()
@ -56,7 +57,7 @@ class TestRFPDupeFilter:
"DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
scheduler = build_from_crawler(Scheduler, crawler)
assert scheduler.df.debug
assert scheduler.df.method == "from_crawler"
@ -65,7 +66,7 @@ class TestRFPDupeFilter:
"DUPEFILTER_CLASS": DirectDupeFilter,
}
crawler = get_crawler(settings_dict=settings)
scheduler = Scheduler.from_crawler(crawler)
scheduler = build_from_crawler(Scheduler, crawler)
assert scheduler.df.method == "n/a"
def test_filter(self):
@ -145,7 +146,7 @@ class TestRFPDupeFilter:
path = tempfile.mkdtemp()
crawler = get_crawler(settings_dict={"JOBDIR": path})
try:
scheduler = Scheduler.from_crawler(crawler)
scheduler = build_from_crawler(Scheduler, crawler)
df = scheduler.df
df.open()
df.request_seen(r1)

3
tests/test_engine.py
View File

@ -23,6 +23,7 @@ from scrapy.utils.defer import (
deferred_from_coro,
maybe_deferred_to_future,
)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.bases.engine import TestEngineBase
@ -287,7 +288,7 @@ async def test_request_scheduled_signal():
crawler = get_crawler(MySpider)
engine = ExecutionEngine(crawler, lambda _: None)
scheduler = TestScheduler() # type: ignore[abstract]
scheduler = build_from_crawler(TestScheduler, crawler)
async def start() -> AsyncIterator[Any]:
return

11
tests/test_extension_debug.py
View File

@ -12,6 +12,7 @@ import pytest
from scrapy.extensions.debug import Debugger, StackTraceDump
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
@ -48,7 +49,7 @@ class SignalSpider(Spider):
)
def test_stacktracedump_installs_signal_handlers() -> None:
crawler = get_crawler()
ext = StackTraceDump.from_crawler(crawler)
ext = build_from_crawler(StackTraceDump, crawler)
assert signal.getsignal(signal.SIGUSR2) == ext.dump_stacktrace # pylint: disable=comparison-with-callable
assert signal.getsignal(signal.SIGQUIT) == ext.dump_stacktrace # pylint: disable=comparison-with-callable
@ -58,15 +59,15 @@ def test_stacktracedump_works_without_signal_support(
) -> None:
# simulate win32 platforms, which don't support SIGUSR signals
monkeypatch.delattr(signal, "SIGUSR2", raising=False)
ext = StackTraceDump.from_crawler(get_crawler())
ext = build_from_crawler(StackTraceDump, get_crawler())
assert isinstance(ext, StackTraceDump)
def test_stacktracedump_dump_stacktrace(caplog: pytest.LogCaptureFixture) -> None:
crawler = get_crawler()
crawler.engine = mock.Mock()
ext = StackTraceDump.from_crawler(crawler)
spider = DefaultSpider()
ext = build_from_crawler(StackTraceDump, crawler)
spider = DefaultSpider.from_crawler(crawler)
with caplog.at_level(logging.INFO, logger="scrapy.extensions.debug"):
ext.dump_stacktrace(0, None)
for r in caplog.records:
@ -82,7 +83,7 @@ def test_stacktracedump_dump_stacktrace(caplog: pytest.LogCaptureFixture) -> Non
def test_stacktracedump_thread_stacks() -> None:
ext = StackTraceDump.from_crawler(get_crawler())
ext = build_from_crawler(StackTraceDump, get_crawler())
stop = threading.Event()
thread = threading.Thread(target=stop.wait, name="dump-test-thread")
thread.start()

7
tests/test_extension_memdebug.py
View File

@ -6,6 +6,7 @@ import pytest
from scrapy.exceptions import NotConfigured
from scrapy.extensions.memdebug import MemoryDebugger
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from scrapy.utils.trackref import object_ref
@ -14,12 +15,12 @@ from tests.utils.decorators import coroutine_test
def test_disabled_by_default() -> None:
with pytest.raises(NotConfigured):
MemoryDebugger.from_crawler(get_crawler())
build_from_crawler(MemoryDebugger, get_crawler())
def test_spider_closed_sets_stats() -> None:
crawler = get_crawler(settings_dict={"MEMDEBUG_ENABLED": True})
ext = MemoryDebugger.from_crawler(crawler)
ext = build_from_crawler(MemoryDebugger, crawler)
class TrackedObject(object_ref):
pass
@ -30,7 +31,7 @@ def test_spider_closed_sets_stats() -> None:
tracked = [TrackedObject(), TrackedObject()]
CollectedObject()
ext.spider_closed(DefaultSpider(), "finished")
ext.spider_closed(DefaultSpider.from_crawler(crawler), "finished")
assert crawler.stats
assert crawler.stats.get_value("memdebug/gc_garbage_count") == len(gc.garbage)

3
tests/test_extension_memusage.py
View File

@ -12,6 +12,7 @@ from scrapy.exceptions import NotConfigured
from scrapy.extensions import memusage as memusage_mod
from scrapy.extensions.memusage import MemoryUsage
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils import OneShotLoop
from tests.utils.cmdline import proc
@ -60,7 +61,7 @@ def test_memusage_disabled() -> None:
"MEMUSAGE_ENABLED": False,
}
with pytest.raises(NotConfigured):
MemoryUsage.from_crawler(get_crawler(settings_dict=settings))
build_from_crawler(MemoryUsage, get_crawler(settings_dict=settings))
def test_memusage_limit_stops_crawler_without_spider(mockserver: MockServer) -> None:

3
tests/test_extension_periodic_log.py
View File

@ -9,6 +9,7 @@ import pytest
from scrapy.exceptions import NotConfigured
from scrapy.extensions.periodic_log import PeriodicLog
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from .spiders import MetaSpider
@ -72,7 +73,7 @@ class CustomPeriodicLog(PeriodicLog):
def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog:
crawler = get_crawler(MetaSpider, settings)
return CustomPeriodicLog.from_crawler(crawler)
return build_from_crawler(CustomPeriodicLog, crawler)
class TestPeriodicLog:

7
tests/test_extension_statsmailer.py
View File

@ -7,6 +7,7 @@ from scrapy import signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.signalmanager import SignalManager
from scrapy.statscollectors import StatsCollector
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
with warnings.catch_warnings():
@ -43,7 +44,7 @@ def test_from_crawler_without_recipients_raises_notconfigured():
crawler.stats = MagicMock()
with pytest.raises(NotConfigured):
statsmailer.StatsMailer.from_crawler(crawler)
build_from_crawler(statsmailer.StatsMailer, crawler)
def test_from_crawler_with_recipients_initializes_extension(dummy_stats, monkeypatch):
@ -55,7 +56,7 @@ def test_from_crawler_with_recipients_initializes_extension(dummy_stats, monkeyp
mailer = MagicMock(spec=MailSender)
monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer)
ext = statsmailer.StatsMailer.from_crawler(crawler)
ext = build_from_crawler(statsmailer.StatsMailer, crawler)
assert isinstance(ext, statsmailer.StatsMailer)
assert ext.recipients == ["test@example.com"]
@ -71,7 +72,7 @@ def test_from_crawler_connects_spider_closed_signal(dummy_stats, monkeypatch):
mailer = MagicMock(spec=MailSender)
monkeypatch.setattr(statsmailer.MailSender, "from_crawler", lambda _: mailer)
statsmailer.StatsMailer.from_crawler(crawler)
build_from_crawler(statsmailer.StatsMailer, crawler)
connected = crawler.signals.send_catch_log(
signals.spider_closed, spider=DefaultSpider(name="dummy")

5
tests/test_extension_telnet.py
View File

@ -11,6 +11,7 @@ from twisted.cred import credentials
from scrapy import Spider
from scrapy.extensions.telnet import TelnetConsole, update_telnet_vars
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
@ -39,7 +40,7 @@ def _get_console_and_portal(
settings: dict[str, Any] | None = None,
) -> Generator[tuple[TelnetConsole, Any]]:
crawler = _get_crawler(settings_dict=settings)
console = TelnetConsole(crawler)
console = build_from_crawler(TelnetConsole, crawler)
# This function has some side effects we don't need for this test
console._get_telnet_vars = dict # type: ignore[method-assign]
@ -87,7 +88,7 @@ async def test_custom_credentials() -> None:
def test_invalid_reversed_portrange() -> None:
settings = {"TELNETCONSOLE_PORT": [2, 1]}
console = TelnetConsole(_get_crawler(settings_dict=settings))
console = build_from_crawler(TelnetConsole, _get_crawler(settings_dict=settings))
with pytest.raises(ValueError, match=r"invalid portrange: \[2, 1\]"):
console.start_listening()

18
tests/test_extension_throttle.py
View File

@ -81,7 +81,7 @@ def test_mindelay_definition(setting, expected):
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
_mock_downloader(crawler)
at._spider_opened(DefaultSpider())
at._spider_opened(DefaultSpider.from_crawler(crawler))
assert at.mindelay == expected
@ -99,7 +99,7 @@ def test_maxdelay_definition(value, expected):
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
_mock_downloader(crawler)
at._spider_opened(DefaultSpider())
at._spider_opened(DefaultSpider.from_crawler(crawler))
assert at.maxdelay == expected
@ -133,7 +133,7 @@ def test_startdelay_definition(min_setting, start_setting, expected):
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
at._spider_opened(DefaultSpider())
at._spider_opened(DefaultSpider.from_crawler(crawler))
assert downloader._delay == expected
@ -159,7 +159,7 @@ def test_skipped(meta, slot):
crawler = get_crawler()
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
spider = DefaultSpider.from_crawler(crawler)
at._spider_opened(spider)
request = Request("https://example.com", meta=meta)
@ -187,7 +187,7 @@ def test_adjustment(download_latency, target_concurrency, slot_delay, expected):
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
spider = DefaultSpider.from_crawler(crawler)
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
@ -221,7 +221,7 @@ def test_adjustment_limits(mindelay, maxdelay, expected):
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
spider = DefaultSpider.from_crawler(crawler)
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
@ -251,7 +251,7 @@ def test_adjustment_bad_response(
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
spider = DefaultSpider.from_crawler(crawler)
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
@ -271,7 +271,7 @@ def test_debug(caplog):
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
spider = DefaultSpider.from_crawler(crawler)
at._spider_opened(spider)
meta = {"download_latency": 1.0, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
@ -299,7 +299,7 @@ def test_debug_disabled(caplog):
crawler = get_crawler()
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
spider = DefaultSpider.from_crawler(crawler)
at._spider_opened(spider)
meta = {"download_latency": 1.0, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)

16
tests/test_feedexport.py
View File

@ -27,6 +27,7 @@ from scrapy.extensions.feedexport import (
ItemFilter,
apply_uri_params,
)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
from tests.spiders import ItemSpider
@ -1262,9 +1263,8 @@ class TestFeedExporterSignals:
feed_slot_signal_handler: Callable[[Any], Awaitable[None] | None],
) -> None:
crawler = get_crawler(settings_dict=self.settings)
feed_exporter = FeedExporter.from_crawler(crawler)
spider = scrapy.Spider("default")
spider.crawler = crawler
feed_exporter = build_from_crawler(FeedExporter, crawler)
spider = scrapy.Spider.from_crawler(crawler, "default")
crawler.signals.connect(
feed_exporter_signal_handler,
signal=signals.feed_exporter_closed,
@ -1318,7 +1318,7 @@ class TestFeedExportInit:
}
crawler = get_crawler(settings_dict=settings)
with pytest.raises(NotConfigured):
FeedExporter.from_crawler(crawler)
build_from_crawler(FeedExporter, crawler)
def test_disabled_storage(self, caplog: pytest.LogCaptureFixture):
class DisabledFeedStorage:
@ -1333,7 +1333,7 @@ class TestFeedExportInit:
}
crawler = get_crawler(settings_dict=settings)
with caplog.at_level(logging.ERROR), pytest.raises(NotConfigured):
FeedExporter.from_crawler(crawler)
build_from_crawler(FeedExporter, crawler)
assert (
"Disabled feed storage scheme: disabled. Reason: not today" in caplog.text
)
@ -1348,7 +1348,7 @@ class TestFeedExportInit:
}
crawler = get_crawler(settings_dict=settings)
with pytest.raises(NotConfigured):
FeedExporter.from_crawler(crawler)
build_from_crawler(FeedExporter, crawler)
def test_absolute_pathlib_as_uri(self):
with tempfile.NamedTemporaryFile(suffix="json") as tmp:
@ -1360,7 +1360,7 @@ class TestFeedExportInit:
},
}
crawler = get_crawler(settings_dict=settings)
exporter = FeedExporter.from_crawler(crawler)
exporter = build_from_crawler(FeedExporter, crawler)
assert isinstance(exporter, FeedExporter)
def test_relative_pathlib_as_uri(self):
@ -1372,7 +1372,7 @@ class TestFeedExportInit:
},
}
crawler = get_crawler(settings_dict=settings)
exporter = FeedExporter.from_crawler(crawler)
exporter = build_from_crawler(FeedExporter, crawler)
assert isinstance(exporter, FeedExporter)

3
tests/test_feedexport_batch.py
View File

@ -18,6 +18,7 @@ from scrapy import Spider
from scrapy.exceptions import NotConfigured
from scrapy.extensions.feedexport import FeedExporter, S3FeedStorage
from scrapy.settings import Settings
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
from tests.spiders import ItemSpider
@ -261,7 +262,7 @@ class TestBatchDeliveries(TestFeedExportBase):
}
crawler = get_crawler(settings_dict=settings)
with pytest.raises(NotConfigured):
FeedExporter(crawler)
build_from_crawler(FeedExporter, crawler)
@coroutine_test
async def test_export_no_items_not_store_empty(self):

39
tests/test_feedexport_storages.py
View File

@ -26,6 +26,7 @@ from scrapy.extensions.feedexport import (
StdoutFeedStorage,
)
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
from tests.utils.cloud import mock_google_cloud_storage
@ -113,7 +114,8 @@ class TestFTPFeedStorage:
settings: dict[str, Any] | None = None,
) -> None:
crawler = get_crawler(settings_dict=settings or {})
storage = FTPFeedStorage.from_crawler(
storage = build_from_crawler(
FTPFeedStorage,
crawler,
uri,
feed_options=feed_options,
@ -249,7 +251,8 @@ class TestS3FeedStorage:
}
crawler = get_crawler(settings_dict=aws_credentials)
# Instantiate with crawler
storage = S3FeedStorage.from_crawler(
storage = build_from_crawler(
S3FeedStorage,
crawler,
"s3://mybucket/export.csv",
)
@ -284,7 +287,7 @@ class TestS3FeedStorage:
crawler = get_crawler(settings_dict=settings)
bucket = "mybucket"
key = "export.csv"
storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}")
storage = build_from_crawler(S3FeedStorage, crawler, f"s3://{bucket}/{key}")
file = mock.MagicMock()
@ -338,7 +341,8 @@ class TestS3FeedStorage:
"AWS_SECRET_ACCESS_KEY": "secret_key",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
storage = build_from_crawler(
S3FeedStorage,
crawler,
"s3://mybucket/export.csv",
)
@ -352,7 +356,8 @@ class TestS3FeedStorage:
"AWS_SECRET_ACCESS_KEY": "secret_key",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
storage = build_from_crawler(
S3FeedStorage,
crawler,
"s3://mybucket/export.csv",
)
@ -366,7 +371,8 @@ class TestS3FeedStorage:
"AWS_SECRET_ACCESS_KEY": "secret_key",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
storage = build_from_crawler(
S3FeedStorage,
crawler,
"s3://mybucket/export.csv",
)
@ -381,7 +387,8 @@ class TestS3FeedStorage:
"FEED_STORAGE_S3_ACL": "custom-acl",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(
storage = build_from_crawler(
S3FeedStorage,
crawler,
"s3://mybucket/export.csv",
)
@ -396,7 +403,7 @@ class TestS3FeedStorage:
"AWS_ENDPOINT_URL": "https://example.com",
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")
storage = build_from_crawler(S3FeedStorage, crawler, "s3://mybucket/export.csv")
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.endpoint_url == "https://example.com"
@ -409,7 +416,7 @@ class TestS3FeedStorage:
"AWS_REGION_NAME": region_name,
}
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")
storage = build_from_crawler(S3FeedStorage, crawler, "s3://mybucket/export.csv")
assert storage.access_key == "access_key"
assert storage.secret_key == "secret_key"
assert storage.region_name == region_name
@ -445,7 +452,7 @@ class TestS3FeedStorage:
self, settings: dict[str, Any], expected: int
) -> None:
crawler = get_crawler(settings_dict=settings)
storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")
storage = build_from_crawler(S3FeedStorage, crawler, "s3://mybucket/export.csv")
assert storage.max_pool_connections == expected
config: Any = storage.s3_client.meta.config
assert config.max_pool_connections == expected
@ -507,7 +514,9 @@ class TestGCSFeedStorage:
settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"}
crawler = get_crawler(settings_dict=settings)
storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
storage = build_from_crawler(
GCSFeedStorage, crawler, "gs://mybucket/export.csv"
)
assert storage.project_id == "123"
assert storage.acl == "publicRead"
assert storage.bucket_name == "mybucket"
@ -518,12 +527,16 @@ class TestGCSFeedStorage:
settings: dict[str, Any] = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""}
crawler = get_crawler(settings_dict=settings)
storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
storage = build_from_crawler(
GCSFeedStorage, crawler, "gs://mybucket/export.csv"
)
assert storage.acl is None
settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": None}
crawler = get_crawler(settings_dict=settings)
storage = GCSFeedStorage.from_crawler(crawler, "gs://mybucket/export.csv")
storage = build_from_crawler(
GCSFeedStorage, crawler, "gs://mybucket/export.csv"
)
assert storage.acl is None
@coroutine_test

15
tests/test_feedexport_uri_params.py
View File

@ -49,8 +49,7 @@ class TestURIParams(ABC):
uri="file:///tmp/%(name)s",
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
spider = scrapy.Spider.from_crawler(crawler, self.spider_name)
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
@ -67,8 +66,7 @@ class TestURIParams(ABC):
uri_params=uri_params,
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
spider = scrapy.Spider.from_crawler(crawler, self.spider_name)
feed_exporter.open_spider(spider)
@ -83,8 +81,7 @@ class TestURIParams(ABC):
uri_params=uri_params,
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
spider = scrapy.Spider.from_crawler(crawler, self.spider_name)
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
@ -100,8 +97,7 @@ class TestURIParams(ABC):
uri_params=uri_params,
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
spider = scrapy.Spider.from_crawler(crawler, self.spider_name)
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)
@ -117,8 +113,7 @@ class TestURIParams(ABC):
uri_params=uri_params,
)
crawler, feed_exporter = self._crawler_feed_exporter(settings)
spider = scrapy.Spider(self.spider_name)
spider.crawler = crawler
spider = scrapy.Spider.from_crawler(crawler, self.spider_name)
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
feed_exporter.open_spider(spider)

16
tests/test_logformatter.py
View File

@ -11,6 +11,7 @@ from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.logformatter import LogFormatter
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.spiders import ItemSpider
from tests.utils.decorators import coroutine_test
@ -28,9 +29,9 @@ class CustomItem(Item):
class TestLogFormatter:
def setup_method(self):
self.formatter = LogFormatter()
self.spider = Spider("default")
self.spider.crawler = get_crawler()
crawler = get_crawler()
self.formatter = build_from_crawler(LogFormatter, crawler)
self.spider = Spider.from_crawler(crawler, "default")
def test_crawled_without_referer(self):
req = Request("http://www.example.com")
@ -75,8 +76,7 @@ class TestLogFormatter:
item = {}
exception = DropItem("Test drop")
response = Response("http://www.example.com")
spider = Spider("foo")
spider.crawler = get_crawler(Spider)
spider = Spider.from_crawler(get_crawler(Spider), "foo")
logkws = self.formatter.dropped(item, exception, response, spider)
assert logkws["level"] == logging.WARNING
@ -198,9 +198,9 @@ class LogFormatterSubclass(LogFormatter):
class TestLogformatterSubclass(TestLogFormatter):
def setup_method(self):
self.formatter = LogFormatterSubclass()
self.spider = Spider("default")
self.spider.crawler = get_crawler(Spider)
crawler = get_crawler(Spider)
self.formatter = build_from_crawler(LogFormatterSubclass, crawler)
self.spider = Spider.from_crawler(crawler, "default")
def test_crawled_without_referer(self):
req = Request("http://www.example.com")

7
tests/test_logstats.py
View File

@ -5,6 +5,7 @@ from datetime import datetime
import pytest
from scrapy.extensions.logstats import LogStats
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
from tests.utils.decorators import coroutine_test
@ -22,7 +23,7 @@ class TestLogStats:
@coroutine_test
async def test_stats_calculations(self) -> None:
logstats = LogStats.from_crawler(self.crawler)
logstats = build_from_crawler(LogStats, self.crawler)
with pytest.raises(AttributeError):
logstats.pagesprev
@ -63,14 +64,14 @@ class TestLogStats:
"""The stat values should be None since the start and finish time are
not available.
"""
logstats = LogStats.from_crawler(self.crawler)
logstats = build_from_crawler(LogStats, self.crawler)
logstats.spider_closed(self.spider, "test reason")
assert self.stats.get_value("responses_per_minute") is None
assert self.stats.get_value("items_per_minute") is None
def test_stats_calculation_no_elapsed_time(self) -> None:
"""The stat values should be None since the elapsed time is 0."""
logstats = LogStats.from_crawler(self.crawler)
logstats = build_from_crawler(LogStats, self.crawler)
self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))
self.stats.set_value("finish_time", datetime.fromtimestamp(1655100172))
logstats.spider_closed(self.spider, "test reason")

3
tests/test_middleware.py
View File

@ -7,6 +7,7 @@ import pytest
from scrapy import Spider
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.middleware import MiddlewareManager
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@ -93,7 +94,7 @@ def test_enabled(crawler: Crawler) -> None:
def test_enabled_from_settings(crawler: Crawler) -> None:
crawler = get_crawler()
mwman = MyMiddlewareManager.from_crawler(crawler)
mwman = build_from_crawler(MyMiddlewareManager, crawler)
classes = [x.__class__ for x in mwman.middlewares]
assert classes == [M1, M3]
assert mwman.crawler == crawler

63
tests/test_pipeline_files.py
View File

@ -39,6 +39,7 @@ from scrapy.pipelines.media import _MediaRequestFiltered
from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.ftp import MockFTPServer
@ -107,7 +108,7 @@ class TestFilesPipeline:
)
crawler.spider = crawler._create_spider()
crawler.engine = MagicMock(download_async=mocked_download_func)
pipeline = pipeline_cls.from_crawler(crawler)
pipeline = build_from_crawler(pipeline_cls, crawler)
pipeline.open_spider()
return pipeline
@ -394,8 +395,8 @@ class TestFilesPipeline:
def file_path(self, request, response=None, info=None, item=None) -> str:
return f"full/{item.get('path')}"
file_path = CustomFilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": self.tempdir})
file_path = build_from_crawler(
CustomFilesPipeline, get_crawler(None, {"FILES_STORE": self.tempdir})
).file_path
item = {"path": "path-to-store-file"}
request = Request("http://example.com")
@ -518,8 +519,8 @@ class TestFilesPipeline:
],
)
def test_rejects_non_list_file_urls(self, tmp_path, bad_type):
pipeline = FilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": str(tmp_path)})
pipeline = build_from_crawler(
FilesPipeline, get_crawler(None, {"FILES_STORE": str(tmp_path)})
)
item = ItemWithFiles()
item["file_urls"] = bad_type
@ -537,8 +538,8 @@ class TestFilesPipelineFieldsMixin(ABC):
def test_item_fields_default(self, tmp_path):
url = "http://www.example.com/files/1.txt"
item = self.item_class(name="item1", file_urls=[url])
pipeline = FilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": tmp_path})
pipeline = build_from_crawler(
FilesPipeline, get_crawler(None, {"FILES_STORE": tmp_path})
)
requests = list(pipeline.get_media_requests(item, None)) # type: ignore[arg-type]
assert requests[0].url == url
@ -551,7 +552,8 @@ class TestFilesPipelineFieldsMixin(ABC):
def test_item_fields_override_settings(self, tmp_path):
url = "http://www.example.com/files/1.txt"
item = self.item_class(name="item1", custom_file_urls=[url])
pipeline = FilesPipeline.from_crawler(
pipeline = build_from_crawler(
FilesPipeline,
get_crawler(
None,
{
@ -559,7 +561,7 @@ class TestFilesPipelineFieldsMixin(ABC):
"FILES_URLS_FIELD": "custom_file_urls",
"FILES_RESULT_FIELD": "custom_files",
},
)
),
)
requests = list(pipeline.get_media_requests(item, None)) # type: ignore[arg-type]
assert requests[0].url == url
@ -664,10 +666,12 @@ class TestFilesPipelineCustomSettings:
different settings.
"""
custom_settings = self._generate_fake_settings(tmp_path)
another_pipeline = FilesPipeline.from_crawler(
get_crawler(None, custom_settings)
another_pipeline = build_from_crawler(
FilesPipeline, get_crawler(None, custom_settings)
)
one_pipeline = build_from_crawler(
FilesPipeline, get_crawler(None, {"FILES_STORE": tmp_path})
)
one_pipeline = FilesPipeline(tmp_path, crawler=get_crawler(None))
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
default_value = self.default_cls_settings[pipe_attr]
assert getattr(one_pipeline, pipe_attr) == default_value
@ -680,7 +684,9 @@ class TestFilesPipelineCustomSettings:
If subclasses override class attributes and there are no special settings those values should be kept.
"""
pipe_cls = self._generate_fake_pipeline()
pipe = pipe_cls.from_crawler(get_crawler(None, {"FILES_STORE": tmp_path}))
pipe = build_from_crawler(
pipe_cls, get_crawler(None, {"FILES_STORE": tmp_path})
)
for pipe_attr, _, pipe_ins_attr in self.file_cls_attr_settings_map:
custom_value = getattr(pipe, pipe_ins_attr)
assert custom_value != self.default_cls_settings[pipe_attr]
@ -693,7 +699,7 @@ class TestFilesPipelineCustomSettings:
"""
pipeline_cls = self._generate_fake_pipeline()
settings = self._generate_fake_settings(tmp_path)
pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
pipeline = build_from_crawler(pipeline_cls, get_crawler(None, settings))
for pipe_attr, settings_attr, pipe_ins_attr in self.file_cls_attr_settings_map:
value = getattr(pipeline, pipe_ins_attr)
setting_value = settings.get(settings_attr)
@ -709,8 +715,8 @@ class TestFilesPipelineCustomSettings:
class UserDefinedFilesPipeline(FilesPipeline):
pass
user_pipeline = UserDefinedFilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": tmp_path})
user_pipeline = build_from_crawler(
UserDefinedFilesPipeline, get_crawler(None, {"FILES_STORE": tmp_path})
)
for pipe_attr, _, pipe_ins_attr in self.file_cls_attr_settings_map:
# Values from settings for custom pipeline should be set on pipeline instance.
@ -728,8 +734,8 @@ class TestFilesPipelineCustomSettings:
prefix = UserDefinedFilesPipeline.__name__.upper()
settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = UserDefinedFilesPipeline.from_crawler(
get_crawler(None, settings)
user_pipeline = build_from_crawler(
UserDefinedFilesPipeline, get_crawler(None, settings)
)
for pipe_attr, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
# Values from settings for custom pipeline should be set on pipeline instance.
@ -745,7 +751,7 @@ class TestFilesPipelineCustomSettings:
pipeline_cls = self._generate_fake_pipeline()
prefix = pipeline_cls.__name__.upper()
settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
user_pipeline = build_from_crawler(pipeline_cls, get_crawler(None, settings))
for (
pipe_cls_attr,
settings_attr,
@ -760,8 +766,8 @@ class TestFilesPipelineCustomSettings:
DEFAULT_FILES_RESULT_FIELD = "this"
DEFAULT_FILES_URLS_FIELD = "that"
pipeline = UserDefinedFilesPipeline.from_crawler(
get_crawler(None, {"FILES_STORE": tmp_path})
pipeline = build_from_crawler(
UserDefinedFilesPipeline, get_crawler(None, {"FILES_STORE": tmp_path})
)
assert (
pipeline.files_result_field
@ -782,7 +788,7 @@ class TestFilesPipelineCustomSettings:
class UserPipe(FilesPipeline):
pass
pipeline_cls = UserPipe.from_crawler(get_crawler(None, settings))
pipeline_cls = build_from_crawler(UserPipe, get_crawler(None, settings))
for _, settings_attr, pipe_inst_attr in self.file_cls_attr_settings_map:
expected_value = settings.get(settings_attr)
@ -793,8 +799,9 @@ class TestFilesPipelineCustomSettings:
def file_path(self, request, response=None, info=None, *, item=None) -> str:
return str(Path("subdir") / Path(request.url).name)
pipeline = CustomFilesPipelineWithPathLikeDir.from_crawler(
get_crawler(None, {"FILES_STORE": tmp_path})
pipeline = build_from_crawler(
CustomFilesPipelineWithPathLikeDir,
get_crawler(None, {"FILES_STORE": tmp_path}),
)
request = Request("http://example.com/image01.jpg")
assert pipeline.file_path(request) == str(Path("subdir/image01.jpg"))
@ -968,7 +975,7 @@ class TestS3FilesStore:
crawler = get_crawler(
settings_dict={"FILES_STORE": "s3://mybucket/prefix/", **settings}
)
store = FilesPipeline.from_crawler(crawler).store
store = build_from_crawler(FilesPipeline, crawler).store
assert isinstance(store, S3FilesStore)
config: Any = store.s3_client.meta.config
assert config.max_pool_connections == expected
@ -1209,7 +1216,7 @@ class TestBuildFromCrawler:
class Pipeline(FilesPipeline):
pass
pipe = Pipeline.from_crawler(self.crawler)
pipe = build_from_crawler(Pipeline, self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert pipe.store
@ -1226,7 +1233,7 @@ class TestBuildFromCrawler:
o._from_crawler_called = True
return o
pipe = Pipeline.from_crawler(self.crawler)
pipe = build_from_crawler(Pipeline, self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert pipe.store
@ -1241,4 +1248,4 @@ def test_files_pipeline_raises_notconfigured_when_files_store_invalid(store):
crawler = get_crawler(settings_dict=dict(settings))
with pytest.raises(NotConfigured):
FilesPipeline.from_crawler(crawler)
build_from_crawler(FilesPipeline, crawler)

56
tests/test_pipeline_images.py
View File

@ -20,6 +20,7 @@ from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.pipelines.files import GCSFilesStore, S3FilesStore, _md5sum
from scrapy.pipelines.images import ImageException, ImagesPipeline
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
from tests.utils.media_pipelines import DUMMY_SPIDER_INFO
@ -40,8 +41,8 @@ else:
class TestImagesPipeline:
def setup_method(self):
self.tempdir = mkdtemp()
crawler = get_crawler()
self.pipeline = ImagesPipeline(self.tempdir, crawler=crawler)
crawler = get_crawler(None, {"IMAGES_STORE": self.tempdir})
self.pipeline = build_from_crawler(ImagesPipeline, crawler)
def teardown_method(self):
rmtree(self.tempdir)
@ -136,8 +137,8 @@ class TestImagesPipeline:
) -> str:
return f"thumb/{thumb_id}/{item.get('path')}"
thumb_path = CustomImagesPipeline.from_crawler(
get_crawler(None, {"IMAGES_STORE": self.tempdir})
thumb_path = build_from_crawler(
CustomImagesPipeline, get_crawler(None, {"IMAGES_STORE": self.tempdir})
).thumb_path
item = {"path": "path-to-store-file"}
request = Request("http://example.com")
@ -301,8 +302,8 @@ class TestImagesPipeline:
],
)
def test_rejects_non_list_image_urls(self, tmp_path, bad_type):
pipeline = ImagesPipeline.from_crawler(
get_crawler(None, {"IMAGES_STORE": str(tmp_path)})
pipeline = build_from_crawler(
ImagesPipeline, get_crawler(None, {"IMAGES_STORE": str(tmp_path)})
)
item = ImagesPipelineTestItem()
item["image_urls"] = bad_type
@ -320,8 +321,8 @@ class TestImagesPipelineFieldsMixin(ABC):
def test_item_fields_default(self):
url = "http://www.example.com/images/1.jpg"
item = self.item_class(name="item1", image_urls=[url])
pipeline = ImagesPipeline.from_crawler(
get_crawler(None, {"IMAGES_STORE": "s3://example/images/"})
pipeline = build_from_crawler(
ImagesPipeline, get_crawler(None, {"IMAGES_STORE": "s3://example/images/"})
)
requests = list(pipeline.get_media_requests(item, DUMMY_SPIDER_INFO))
assert requests[0].url == url
@ -334,7 +335,8 @@ class TestImagesPipelineFieldsMixin(ABC):
def test_item_fields_override_settings(self):
url = "http://www.example.com/images/1.jpg"
item = self.item_class(name="item1", custom_image_urls=[url])
pipeline = ImagesPipeline.from_crawler(
pipeline = build_from_crawler(
ImagesPipeline,
get_crawler(
None,
{
@ -342,7 +344,7 @@ class TestImagesPipelineFieldsMixin(ABC):
"IMAGES_URLS_FIELD": "custom_image_urls",
"IMAGES_RESULT_FIELD": "custom_images",
},
)
),
)
requests = list(pipeline.get_media_requests(item, DUMMY_SPIDER_INFO))
assert requests[0].url == url
@ -478,8 +480,12 @@ class TestImagesPipelineCustomSettings:
have different settings.
"""
custom_settings = self._generate_fake_settings(tmp_path)
default_sts_pipe = ImagesPipeline(tmp_path, crawler=get_crawler(None))
user_sts_pipe = ImagesPipeline.from_crawler(get_crawler(None, custom_settings))
default_sts_pipe = build_from_crawler(
ImagesPipeline, get_crawler(None, {"IMAGES_STORE": tmp_path})
)
user_sts_pipe = build_from_crawler(
ImagesPipeline, get_crawler(None, custom_settings)
)
for pipe_attr, settings_attr in self.img_cls_attribute_names:
expected_default_value = self.default_pipeline_settings.get(pipe_attr)
custom_value = custom_settings.get(settings_attr)
@ -495,8 +501,8 @@ class TestImagesPipelineCustomSettings:
from class attributes.
"""
pipeline_cls = self._generate_fake_pipeline_subclass()
pipeline = pipeline_cls.from_crawler(
get_crawler(None, {"IMAGES_STORE": tmp_path})
pipeline = build_from_crawler(
pipeline_cls, get_crawler(None, {"IMAGES_STORE": tmp_path})
)
for pipe_attr, _ in self.img_cls_attribute_names:
# Instance attribute (lowercase) must be equal to class attribute (uppercase).
@ -511,7 +517,7 @@ class TestImagesPipelineCustomSettings:
"""
pipeline_cls = self._generate_fake_pipeline_subclass()
settings = self._generate_fake_settings(tmp_path)
pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
pipeline = build_from_crawler(pipeline_cls, get_crawler(None, settings))
for pipe_attr, settings_attr in self.img_cls_attribute_names:
# Instance attribute (lowercase) must be equal to
# value defined in settings.
@ -529,8 +535,8 @@ class TestImagesPipelineCustomSettings:
class UserDefinedImagePipeline(ImagesPipeline):
pass
user_pipeline = UserDefinedImagePipeline.from_crawler(
get_crawler(None, {"IMAGES_STORE": tmp_path})
user_pipeline = build_from_crawler(
UserDefinedImagePipeline, get_crawler(None, {"IMAGES_STORE": tmp_path})
)
for pipe_attr, _ in self.img_cls_attribute_names:
# Values from settings for custom pipeline should be set on pipeline instance.
@ -548,8 +554,8 @@ class TestImagesPipelineCustomSettings:
prefix = UserDefinedImagePipeline.__name__.upper()
settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = UserDefinedImagePipeline.from_crawler(
get_crawler(None, settings)
user_pipeline = build_from_crawler(
UserDefinedImagePipeline, get_crawler(None, settings)
)
for pipe_attr, settings_attr in self.img_cls_attribute_names:
# Values from settings for custom pipeline should be set on pipeline instance.
@ -565,7 +571,7 @@ class TestImagesPipelineCustomSettings:
pipeline_cls = self._generate_fake_pipeline_subclass()
prefix = pipeline_cls.__name__.upper()
settings = self._generate_fake_settings(tmp_path, prefix=prefix)
user_pipeline = pipeline_cls.from_crawler(get_crawler(None, settings))
user_pipeline = build_from_crawler(pipeline_cls, get_crawler(None, settings))
for pipe_attr, settings_attr in self.img_cls_attribute_names:
custom_value = settings.get(prefix + "_" + settings_attr)
assert custom_value != self.default_pipeline_settings[pipe_attr]
@ -576,8 +582,8 @@ class TestImagesPipelineCustomSettings:
DEFAULT_IMAGES_URLS_FIELD = "something"
DEFAULT_IMAGES_RESULT_FIELD = "something_else"
pipeline = UserDefinedImagePipeline.from_crawler(
get_crawler(None, {"IMAGES_STORE": tmp_path})
pipeline = build_from_crawler(
UserDefinedImagePipeline, get_crawler(None, {"IMAGES_STORE": tmp_path})
)
assert (
pipeline.images_result_field
@ -598,7 +604,7 @@ class TestImagesPipelineCustomSettings:
class UserPipe(ImagesPipeline):
pass
pipeline_cls = UserPipe.from_crawler(get_crawler(None, settings))
pipeline_cls = build_from_crawler(UserPipe, get_crawler(None, settings))
for pipe_attr, settings_attr in self.img_cls_attribute_names:
expected_value = settings.get(settings_attr)
@ -617,7 +623,7 @@ class TestImagesPipelineCustomSettings:
},
)
ImagesPipeline.from_crawler(crawler)
build_from_crawler(ImagesPipeline, crawler)
assert S3FilesStore.POLICY == "public-read"
finally:
@ -636,7 +642,7 @@ class TestImagesPipelineCustomSettings:
},
)
ImagesPipeline.from_crawler(crawler)
build_from_crawler(ImagesPipeline, crawler)
assert GCSFilesStore.POLICY == "authenticatedRead"
finally:

9
tests/test_pipeline_media.py
View File

@ -19,6 +19,7 @@ from scrapy.pipelines.media import (
)
from scrapy.utils.defer import _defer_sleep_async
from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.signal import disconnect_all
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
@ -78,7 +79,7 @@ class TestBaseMediaPipeline:
crawler = get_crawler(DefaultSpider, self.settings)
crawler.spider = crawler._create_spider()
crawler.engine = MagicMock(download_async=mocked_download_func)
self.pipe = self.pipeline_class.from_crawler(crawler)
self.pipe = build_from_crawler(self.pipeline_class, crawler)
self.pipe.open_spider()
self.info = self.pipe.spiderinfo
assert crawler.request_fingerprinter is not None
@ -529,7 +530,7 @@ class TestBuildFromCrawler:
class Pipeline(UserDefinedPipeline):
pass
pipe = Pipeline.from_crawler(self.crawler)
pipe = build_from_crawler(Pipeline, self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
@ -549,7 +550,7 @@ class TestBuildFromCrawler:
o._from_crawler_called = True
return o
pipe = Pipeline.from_crawler(self.crawler)
pipe = build_from_crawler(Pipeline, self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert pipe._from_crawler_called
@ -568,7 +569,7 @@ class TestBuildFromCrawler:
o.store_uri = settings["FILES_STORE"]
return o
pipe = Pipeline.from_crawler(self.crawler)
pipe = build_from_crawler(Pipeline, self.crawler)
assert pipe.crawler == self.crawler
assert pipe._fingerprinter
assert pipe._from_crawler_called

3
tests/test_pipelines.py
View File

@ -11,6 +11,7 @@ from scrapy.pipelines import ItemPipelineManager
from scrapy.utils.asyncio import call_later
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.mockserver.http import MockServer
@ -258,7 +259,7 @@ class TestCustomPipelineManager:
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
itemproc = CustomPipelineManager.from_crawler(crawler)
itemproc = build_from_crawler(CustomPipelineManager, crawler)
with pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager.process_item\(\) is deprecated, use process_item_async\(\)",

32
tests/test_pqueues.py
View File

@ -21,8 +21,8 @@ class TestPriorityQueue:
def test_queue_push_pop_one(self):
temp_dir = tempfile.mkdtemp()
queue = ScrapyPriorityQueue.from_crawler(
self.crawler, FifoMemoryQueue, temp_dir
queue = build_from_crawler(
ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir
)
assert queue.pop() is None
assert len(queue) == 0
@ -39,8 +39,8 @@ class TestPriorityQueue:
if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
pytest.skip("queuelib.queue.FifoMemoryQueue.peek is defined")
temp_dir = tempfile.mkdtemp()
queue = ScrapyPriorityQueue.from_crawler(
self.crawler, FifoMemoryQueue, temp_dir
queue = build_from_crawler(
ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir
)
queue.push(Request("https://example.org"))
with pytest.raises(
@ -54,8 +54,8 @@ class TestPriorityQueue:
if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
pytest.skip("queuelib.queue.FifoMemoryQueue.peek is undefined")
temp_dir = tempfile.mkdtemp()
queue = ScrapyPriorityQueue.from_crawler(
self.crawler, FifoMemoryQueue, temp_dir
queue = build_from_crawler(
ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir
)
assert len(queue) == 0
assert queue.peek() is None
@ -78,7 +78,8 @@ class TestPriorityQueue:
def test_init_prios_with_start_queue(self):
temp_dir = tempfile.mkdtemp()
queue = ScrapyPriorityQueue.from_crawler(
queue = build_from_crawler(
ScrapyPriorityQueue,
self.crawler,
PickleFifoDiskQueue,
temp_dir,
@ -88,7 +89,8 @@ class TestPriorityQueue:
queue.push(req)
startprios = queue.close()
queue2 = ScrapyPriorityQueue.from_crawler(
queue2 = build_from_crawler(
ScrapyPriorityQueue,
self.crawler,
PickleFifoDiskQueue,
temp_dir,
@ -101,8 +103,8 @@ class TestPriorityQueue:
def test_queue_push_pop_priorities(self):
temp_dir = tempfile.mkdtemp()
queue = ScrapyPriorityQueue.from_crawler(
self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3]
queue = build_from_crawler(
ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir, [-1, -2, -3]
)
assert queue.pop() is None
assert len(queue) == 0
@ -124,8 +126,9 @@ class TestDownloaderAwarePriorityQueue:
def setup_method(self):
crawler = get_crawler(Spider)
crawler.engine = Mock(downloader=MockDownloader())
self.queue = DownloaderAwarePriorityQueue.from_crawler(
crawler=crawler,
self.queue = build_from_crawler(
DownloaderAwarePriorityQueue,
crawler,
downstream_queue_cls=FifoMemoryQueue,
key="foo/bar",
)
@ -262,8 +265,9 @@ def test_slot_directory_removed_when_slot_drains(tmp_path):
crawler = get_crawler(Spider)
crawler.spider = crawler._create_spider("foo")
crawler.engine = Mock(downloader=MockDownloader())
queue = DownloaderAwarePriorityQueue.from_crawler(
crawler=crawler,
queue = build_from_crawler(
DownloaderAwarePriorityQueue,
crawler,
downstream_queue_cls=PickleFifoDiskQueue,
key=str(tmp_path),
)

5
tests/test_resolver.py
View File

@ -7,6 +7,7 @@ from twisted.internet.address import IPv4Address, IPv6Address
from scrapy.resolver import CachingHostnameResolver, CachingThreadedResolver, dnscache
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
@ -22,7 +23,7 @@ def reset_dnscache():
def test_caching_threaded_resolver_dnscache_disabled():
crawler = get_crawler(settings_dict={"DNSCACHE_ENABLED": False})
CachingThreadedResolver.from_crawler(crawler, Mock())
build_from_crawler(CachingThreadedResolver, crawler, Mock())
assert dnscache.limit == 0
@ -37,7 +38,7 @@ async def test_caching_threaded_resolver_getHostByName_cache_hit():
def test_caching_hostname_resolver_dnscache_disabled():
crawler = get_crawler(settings_dict={"DNSCACHE_ENABLED": False})
CachingHostnameResolver.from_crawler(crawler, Mock())
build_from_crawler(CachingHostnameResolver, crawler, Mock())
assert dnscache.limit == 0

13
tests/test_robotstxt_interface.py
View File

@ -1,6 +1,6 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, cast
import pytest
@ -12,6 +12,7 @@ from scrapy.robotstxt import (
decode_robotstxt,
)
from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
from scrapy.utils.misc import build_from_crawler
from tests.utils.robotstxt import rerp_available
if TYPE_CHECKING:
@ -20,6 +21,9 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
# The parser backends only use the crawler to get the spider to log with.
NO_CRAWLER = cast("Crawler", None)
class BaseRobotParserTest:
parser_cls: type[RobotParser]
@ -28,8 +32,7 @@ class BaseRobotParserTest:
self.parser_cls = parser_cls
def _parse(self, robotstxt_body: bytes) -> RobotParser:
# The parser backends only use the crawler to get the spider to log with.
return self.parser_cls.from_crawler(None, robotstxt_body) # type: ignore[arg-type]
return build_from_crawler(self.parser_cls, NO_CRAWLER, robotstxt_body)
def test_allowed(self):
robotstxt_robotstxt_body = (
@ -127,7 +130,9 @@ class TestRobotParser:
def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
return True
rp = AllowAllRobotParser()
rp = build_from_crawler(
AllowAllRobotParser, NO_CRAWLER, b"User-agent: *\nCrawl-delay: 10\n"
)
assert rp.crawl_delay("*") is None

10
tests/test_scheduler.py
View File

@ -15,7 +15,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
from scrapy.spiders import Spider
from scrapy.utils.defer import ensure_awaitable
from scrapy.utils.misc import load_object
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import coroutine_test, inline_callbacks_test
@ -46,8 +46,8 @@ async def create_scheduler(
priority_queue_cls: str, jobdir: Path | None
) -> AsyncGenerator[Scheduler]:
mock_crawler = MockCrawler(priority_queue_cls, jobdir)
scheduler = Scheduler.from_crawler(mock_crawler)
spider = Spider(name="spider")
scheduler = build_from_crawler(Scheduler, mock_crawler)
spider = Spider.from_crawler(mock_crawler, name="spider")
await ensure_awaitable(scheduler.open(spider))
try:
yield scheduler
@ -333,8 +333,8 @@ class TestIncompatibility:
"CONCURRENT_REQUESTS_PER_IP": 1,
}
crawler = get_crawler(Spider, settings)
scheduler = Scheduler.from_crawler(crawler)
spider = Spider(name="spider")
scheduler = build_from_crawler(Scheduler, crawler)
spider = Spider.from_crawler(crawler, name="spider")
scheduler.open(spider)
def test_incompatibility(self):

11
tests/test_spidermiddleware.py
View File

@ -15,6 +15,7 @@ from scrapy.spiders import Spider
from scrapy.utils.asyncgen import collect_asyncgen
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
@ -31,7 +32,7 @@ class TestSpiderMiddleware:
self.response = Response(self.request.url, request=self.request)
self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES_BASE": {}})
self.crawler.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
self.mwman = build_from_crawler(SpiderMiddlewareManager, self.crawler)
async def _scrape_response(self) -> Any:
"""Execute spider mw manager's scrape_response_async method and return the result.
@ -139,7 +140,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
Spider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting}
)
self.crawler.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
self.mwman = build_from_crawler(SpiderMiddlewareManager, self.crawler)
return await self.mwman.scrape_response_async(
self._scrape_func, self.response, self.request
)
@ -266,7 +267,7 @@ class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware):
TestSpider, {"SPIDER_MIDDLEWARES_BASE": {}, "SPIDER_MIDDLEWARES": setting}
)
self.crawler.spider = self.crawler._create_spider()
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
self.mwman = build_from_crawler(SpiderMiddlewareManager, self.crawler)
return await self.mwman.process_start()
@coroutine_test
@ -307,7 +308,7 @@ class TestUniversalMiddlewareManager:
@pytest.fixture
def mwman(self, crawler: Crawler) -> SpiderMiddlewareManager:
return SpiderMiddlewareManager.from_crawler(crawler)
return build_from_crawler(SpiderMiddlewareManager, crawler)
def test_simple_mw(self, mwman: SpiderMiddlewareManager) -> None:
mw = ProcessSpiderOutputSyncMiddleware()
@ -376,7 +377,7 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES": setting})
self.crawler.spider = self.crawler._create_spider("foo")
self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
self.mwman = build_from_crawler(SpiderMiddlewareManager, self.crawler)
return await self.mwman.scrape_response_async(
self._scrape_func, self.response, self.request
)

9
tests/test_spidermiddleware_base.py
View File

@ -8,6 +8,7 @@ from scrapy import Request, Spider
from scrapy.http import Response
from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
@ -25,7 +26,7 @@ async def test_trivial(crawler: Crawler) -> None:
class TrivialSpiderMiddleware(BaseSpiderMiddleware):
pass
mw = TrivialSpiderMiddleware.from_crawler(crawler)
mw = build_from_crawler(TrivialSpiderMiddleware, crawler)
assert hasattr(mw, "crawler")
assert mw.crawler is crawler
test_req = Request("data:,")
@ -49,7 +50,7 @@ async def test_processed_request(crawler: Crawler) -> None:
return Request("data:30,")
return request
mw = ProcessReqSpiderMiddleware.from_crawler(crawler)
mw = build_from_crawler(ProcessReqSpiderMiddleware, crawler)
test_req1 = Request("data:1,")
test_req2 = Request("data:2,")
test_req3 = Request("data:3,")
@ -81,7 +82,7 @@ async def test_processed_item(crawler: Crawler) -> None:
item["foo"] = 30
return item
mw = ProcessItemSpiderMiddleware.from_crawler(crawler)
mw = build_from_crawler(ProcessItemSpiderMiddleware, crawler)
test_req = Request("data:,")
spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]
for processed in [
@ -110,7 +111,7 @@ async def test_processed_both(crawler: Crawler) -> None:
item["foo"] = 30
return item
mw = ProcessBothSpiderMiddleware.from_crawler(crawler)
mw = build_from_crawler(ProcessBothSpiderMiddleware, crawler)
test_req1 = Request("data:1,")
test_req2 = Request("data:2,")
test_req3 = Request("data:3,")

2
tests/test_spidermiddleware_depth.py
View File

@ -36,7 +36,7 @@ def stats(crawler: Crawler) -> Generator[StatsCollector]:
@pytest.fixture
def mw(crawler: Crawler) -> DepthMiddleware:
return DepthMiddleware.from_crawler(crawler)
return build_from_crawler(DepthMiddleware, crawler)
def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> None:

9
tests/test_spidermiddleware_httperror.py
View File

@ -7,6 +7,7 @@ import pytest
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.spiders import MockServerSpider
@ -79,7 +80,7 @@ class TestHttpErrorMiddleware:
def mw(self) -> HttpErrorMiddleware:
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
return HttpErrorMiddleware.from_crawler(crawler)
return build_from_crawler(HttpErrorMiddleware, crawler)
def test_process_spider_input(
self, mw: HttpErrorMiddleware, res200: Response, res404: Response
@ -115,7 +116,7 @@ class TestHttpErrorMiddlewareSettings:
def mw(self) -> HttpErrorMiddleware:
crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOWED_CODES": (402,)})
crawler.spider = crawler._create_spider()
return HttpErrorMiddleware.from_crawler(crawler)
return build_from_crawler(HttpErrorMiddleware, crawler)
def test_process_spider_input(
self,
@ -155,7 +156,7 @@ class TestHttpErrorMiddlewareHandleAll:
def mw(self) -> HttpErrorMiddleware:
crawler = get_crawler(DefaultSpider, {"HTTPERROR_ALLOW_ALL": True})
crawler.spider = crawler._create_spider()
return HttpErrorMiddleware.from_crawler(crawler)
return build_from_crawler(HttpErrorMiddleware, crawler)
def test_process_spider_input(
self,
@ -179,7 +180,7 @@ class TestHttpErrorMiddlewareHandleAll:
def test_httperror_allow_all_false(self) -> None:
crawler = get_crawler(_HttpErrorSpider)
mw = HttpErrorMiddleware.from_crawler(crawler)
mw = build_from_crawler(HttpErrorMiddleware, crawler)
request_httpstatus_false = Request(
"http://scrapytest.org", meta={"handle_httpstatus_all": False}
)

7
tests/test_spidermiddleware_metacopy.py
View File

@ -8,6 +8,7 @@ import pytest
from scrapy.http import Request, Response
from scrapy.spidermiddlewares.metacopy import MetaCopyDetectionMiddleware
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@ -27,7 +28,7 @@ def crawler() -> Crawler:
@pytest.fixture
def mw(crawler: Crawler) -> MetaCopyDetectionMiddleware:
return MetaCopyDetectionMiddleware.from_crawler(crawler)
return build_from_crawler(MetaCopyDetectionMiddleware, crawler)
def process(
@ -94,7 +95,7 @@ class TestInternalKeysCheck:
def test_skip_keys_setting(self, caplog: pytest.LogCaptureFixture) -> None:
with caplog.at_level(WARNING):
crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})
mw = MetaCopyDetectionMiddleware.from_crawler(crawler)
mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)
req = Request("https://example.com/1", meta={"retry_times": 1})
process(mw, [req])
assert not caplog.records
@ -102,7 +103,7 @@ class TestInternalKeysCheck:
def test_skip_keys_setting_partial(self, caplog: pytest.LogCaptureFixture) -> None:
with caplog.at_level(WARNING):
crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})
mw = MetaCopyDetectionMiddleware.from_crawler(crawler)
mw = build_from_crawler(MetaCopyDetectionMiddleware, crawler)
req = Request(
"https://example.com/1",
meta={"retry_times": 1, "redirect_times": 2},

2
tests/test_spidermiddleware_urllength.py
View File

@ -37,7 +37,7 @@ def stats(crawler: Crawler) -> StatsCollector:
@pytest.fixture
def mw(crawler: Crawler) -> UrlLengthMiddleware:
return UrlLengthMiddleware.from_crawler(crawler)
return build_from_crawler(UrlLengthMiddleware, crawler)
def process_spider_output(mw: UrlLengthMiddleware) -> list[Request]:

3
tests/test_spiderstate.py
View File

@ -8,6 +8,7 @@ import pytest
from scrapy.exceptions import NotConfigured
from scrapy.extensions.spiderstate import SpiderState
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@ -49,4 +50,4 @@ def test_state_attribute() -> None:
def test_not_configured() -> None:
crawler = get_crawler(Spider)
with pytest.raises(NotConfigured):
SpiderState.from_crawler(crawler)
build_from_crawler(SpiderState, crawler)

21
tests/test_squeues_request.py
View File

@ -20,6 +20,7 @@ from scrapy.squeues import (
PickleFifoDiskQueue,
PickleLifoDiskQueue,
)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
@ -114,8 +115,8 @@ class TestPickleFifoDiskQueueRequest(TestRequestQueueBase):
@pytest.fixture
def q(self, crawler, tmp_path):
queue = PickleFifoDiskQueue.from_crawler(
crawler=crawler, key=str(tmp_path / "pickle" / "fifo")
queue = build_from_crawler(
PickleFifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "fifo")
)
try:
yield queue
@ -128,8 +129,8 @@ class TestPickleLifoDiskQueueRequest(TestRequestQueueBase):
@pytest.fixture
def q(self, crawler, tmp_path):
queue = PickleLifoDiskQueue.from_crawler(
crawler=crawler, key=str(tmp_path / "pickle" / "lifo")
queue = build_from_crawler(
PickleLifoDiskQueue, crawler, key=str(tmp_path / "pickle" / "lifo")
)
try:
yield queue
@ -142,8 +143,8 @@ class TestMarshalFifoDiskQueueRequest(TestRequestQueueBase):
@pytest.fixture
def q(self, crawler, tmp_path):
queue = MarshalFifoDiskQueue.from_crawler(
crawler=crawler, key=str(tmp_path / "marshal" / "fifo")
queue = build_from_crawler(
MarshalFifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "fifo")
)
try:
yield queue
@ -156,8 +157,8 @@ class TestMarshalLifoDiskQueueRequest(TestRequestQueueBase):
@pytest.fixture
def q(self, crawler, tmp_path):
queue = MarshalLifoDiskQueue.from_crawler(
crawler=crawler, key=str(tmp_path / "marshal" / "lifo")
queue = build_from_crawler(
MarshalLifoDiskQueue, crawler, key=str(tmp_path / "marshal" / "lifo")
)
try:
yield queue
@ -170,7 +171,7 @@ class TestFifoMemoryQueueRequest(TestRequestQueueBase):
@pytest.fixture
def q(self, crawler):
return FifoMemoryQueue.from_crawler(crawler=crawler)
return build_from_crawler(FifoMemoryQueue, crawler)
class TestLifoMemoryQueueRequest(TestRequestQueueBase):
@ -178,4 +179,4 @@ class TestLifoMemoryQueueRequest(TestRequestQueueBase):
@pytest.fixture
def q(self, crawler):
return LifoMemoryQueue.from_crawler(crawler=crawler)
return build_from_crawler(LifoMemoryQueue, crawler)

5
tests/test_stats.py
View File

@ -10,6 +10,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.corestats import CoreStats
from scrapy.spiders import Spider
from scrapy.statscollectors import DummyStatsCollector, StatsCollector
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
from tests.utils.decorators import coroutine_test
@ -41,7 +42,7 @@ class TestCoreStatsExtension:
fixed_datetime = datetime(2019, 12, 1, 11, 38)
mock_datetime.now = mock.Mock(return_value=fixed_datetime)
crawler.stats = StatsCollector(crawler)
ext = CoreStats.from_crawler(crawler)
ext = build_from_crawler(CoreStats, crawler)
ext.spider_opened(spider)
ext.item_scraped({}, spider)
ext.response_received(spider)
@ -62,7 +63,7 @@ class TestCoreStatsExtension:
self, crawler: Crawler, spider: Spider
) -> None:
crawler.stats = DummyStatsCollector(crawler)
ext = CoreStats.from_crawler(crawler)
ext = build_from_crawler(CoreStats, crawler)
ext.spider_opened(spider)
ext.item_scraped({}, spider)
ext.response_received(spider)

20
tests/test_utils_log.py
View File

@ -4,12 +4,14 @@ import json
import logging
import re
import sys
import warnings
from io import StringIO
from typing import TYPE_CHECKING, Any, cast
import pytest
from twisted.python.failure import Failure
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.log import (
LogCounterHandler,
SpiderLoggerAdapter,
@ -332,7 +334,9 @@ class TestLogformatterAdapter:
"LogFormatterResult",
{"level": logging.INFO, "msg": "90% done", "args": args},
)
assert self._log(caplog, logkws) == "90% done"
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
assert self._log(caplog, logkws) == "90% done"
@pytest.mark.parametrize(
("msg", "args"),
@ -345,4 +349,16 @@ class TestLogformatterAdapter:
args: dict[str, Any] | tuple[Any, ...],
) -> None:
logkws: LogFormatterResult = {"level": logging.INFO, "msg": msg, "args": args}
assert self._log(caplog, logkws) == "90% done"
with warnings.catch_warnings():
warnings.simplefilter("error", ScrapyDeprecationWarning)
assert self._log(caplog, logkws) == "90% done"
def test_msg_mapping_placeholders_without_args(
self, caplog: pytest.LogCaptureFixture
) -> None:
logkws = cast(
"LogFormatterResult",
{"level": logging.INFO, "msg": "%(pct)d%% done", "pct": 90},
)
with pytest.warns(ScrapyDeprecationWarning, match="no args"):
assert self._log(caplog, logkws) == "90% done"

74
tests/utils/bases/redirect.py
View File

@ -8,7 +8,7 @@ import pytest
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
from scrapy.exceptions import IgnoreRequest
from scrapy.http import Request, Response
from scrapy.utils.misc import set_environ
from scrapy.utils.misc import build_from_crawler, set_environ
from scrapy.utils.test import get_crawler
@ -221,8 +221,8 @@ class TestRedirectBase(ABC):
def test_meta_proxy_http_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("http://example.com", meta=meta)
@ -262,8 +262,8 @@ class TestRedirectBase(ABC):
def test_meta_proxy_http_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("http://example.com", meta=meta)
@ -303,8 +303,8 @@ class TestRedirectBase(ABC):
def test_meta_proxy_https_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("https://example.com", meta=meta)
@ -344,8 +344,8 @@ class TestRedirectBase(ABC):
def test_meta_proxy_https_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("https://example.com", meta=meta)
@ -385,8 +385,8 @@ class TestRedirectBase(ABC):
def test_meta_proxy_http_to_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("http://example.com", meta=meta)
@ -426,8 +426,8 @@ class TestRedirectBase(ABC):
def test_meta_proxy_https_to_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
meta = {"proxy": "https://a:@a.example"}
request1 = Request("https://example.com", meta=meta)
@ -467,12 +467,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_http_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
@ -511,12 +511,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_http_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
@ -555,12 +555,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_https_absolute(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"https_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
@ -599,12 +599,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_https_relative(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"https_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
@ -643,13 +643,13 @@ class TestRedirectBase(ABC):
def test_system_proxy_proxied_http_to_proxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"http_proxy": "https://a:@a.example",
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
@ -688,12 +688,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_proxied_http_to_unproxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
@ -732,12 +732,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_unproxied_http_to_proxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
@ -776,8 +776,8 @@ class TestRedirectBase(ABC):
def test_system_proxy_unproxied_http_to_unproxied_https(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("http://example.com")
proxy_mw.process_request(request1)
@ -816,13 +816,13 @@ class TestRedirectBase(ABC):
def test_system_proxy_proxied_https_to_proxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"http_proxy": "https://a:@a.example",
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
@ -861,12 +861,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_proxied_https_to_unproxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"https_proxy": "https://b:@b.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
@ -905,12 +905,12 @@ class TestRedirectBase(ABC):
def test_system_proxy_unproxied_https_to_proxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
env = {
"http_proxy": "https://a:@a.example",
}
with set_environ(**env):
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)
@ -949,8 +949,8 @@ class TestRedirectBase(ABC):
def test_system_proxy_unproxied_https_to_unproxied_http(self):
crawler = get_crawler()
redirect_mw = self.mwcls.from_crawler(crawler)
proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
redirect_mw = build_from_crawler(self.mwcls, crawler)
proxy_mw = build_from_crawler(HttpProxyMiddleware, crawler)
request1 = Request("https://example.com")
proxy_mw.process_request(request1)