RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into active-downloads-maxsize

This commit is contained in:
Adrian Chaves 2026-06-19 10:23:40 +02:00
parent b8d47f74c1 3f3cb885ed
commit 37005262b6
23 changed files with 419 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
conftest.py
View File

@ -74,40 +74,19 @@ def mockserver() -> Generator[MockServer]:
@pytest.fixture # function scope because it modifies os.environ
def mitm_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
proxy = MitmProxy()
def proxy_server(
request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch
) -> Generator[str]:
kind = request.param
proxy = MitmProxy(mode="socks5" if kind == "socks5" else None)
url = proxy.start()
if kind == "https":
url = url.replace("http://", "https://")
monkeypatch.setenv("http_proxy", url)
monkeypatch.setenv("https_proxy", url)
try:
yield proxy
finally:
proxy.stop()
@pytest.fixture # function scope because it modifies os.environ
def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
proxy = MitmProxy()
url = proxy.start().replace("http://", "https://")
monkeypatch.setenv("http_proxy", url)
monkeypatch.setenv("https_proxy", url)
try:
yield proxy
finally:
proxy.stop()
@pytest.fixture # function scope because it modifies os.environ
def socks5_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
proxy = MitmProxy(mode="socks5")
url = proxy.start()
monkeypatch.setenv("http_proxy", url)
monkeypatch.setenv("https_proxy", url)
try:
yield proxy
yield kind
finally:
proxy.stop()

19
scrapy/commands/__init__.py
View File

@ -16,6 +16,8 @@ from twisted.python import failure
from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
from scrapy.utils.deprecate import method_is_overridden
from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
from collections.abc import Iterable
@ -36,6 +38,14 @@ class ScrapyCommand(ABC):
def __init__(self) -> None:
self.settings: Settings | None = None # set in scrapy.cmdline
if method_is_overridden(self.__class__, ScrapyCommand, "help"):
warnings.warn(
"The ScrapyCommand.help() method is deprecated and overriding "
f"it, as the {global_object_name(self.__class__)} class does, "
"has no effect; override long_desc() instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
def set_crawler(self, crawler: Crawler) -> None: # pragma: no cover
warnings.warn(
@ -68,10 +78,11 @@ class ScrapyCommand(ABC):
return self.short_desc()
def help(self) -> str:
"""An extensive help for the command. It will be shown when using the
"help" command. It can contain newlines since no post-formatting will
be applied to its contents.
"""
warnings.warn(
"ScrapyCommand.help() is deprecated, use long_desc() instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return self.long_desc()
def add_options(self, parser: argparse.ArgumentParser) -> None:

4
scrapy/contracts/__init__.py
View File

@ -51,6 +51,8 @@ class Contract:
results.addSuccess(self.testcase_pre)
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
if isinstance(cb_result, CoroutineType):
cb_result.close()
raise TypeError("Contracts don't support async callbacks")
return list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
@ -67,6 +69,8 @@ class Contract:
def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]:
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
if isinstance(cb_result, CoroutineType):
cb_result.close()
raise TypeError("Contracts don't support async callbacks")
output = list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
try:

16
scrapy/core/downloader/handlers/_base_streaming.py
View File

@ -241,6 +241,16 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon
body=response_body.getvalue(),
)
@staticmethod
def _request_headers(request: Request) -> Headers:
"""Get a prepared copy of the request headers.
This removes the Proxy-Authorization header.
"""
headers = request.headers.copy()
headers.pop(b"Proxy-Authorization", None)
return headers
def _get_bind_address_host(self) -> str | None:
"""Return the host portion of the bind address.
@ -279,10 +289,8 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon
if not proxy:
return None, None
proxy = add_http_if_no_scheme(proxy)
auth_header: list[bytes] | None = request.headers.pop(
b"Proxy-Authorization", None
)
return proxy, auth_header[0].decode("ascii") if auth_header else None
auth_header: bytes | None = request.headers.get(b"Proxy-Authorization")
return proxy, auth_header.decode("ascii") if auth_header else None
def _extract_proxy_url_with_creds(self, request: Request) -> str | None:
"""Return the proxy URL with the userinfo added based on the

3
scrapy/core/downloader/handlers/_httpx.py
View File

@ -152,13 +152,14 @@ class HttpxDownloadHandler(_Base):
f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed."
)
client = self._get_client(proxy)
headers = self._request_headers(request).to_tuple_list()
try:
async with client.stream(
request.method,
request.url,
content=request.body,
headers=request.headers.to_tuple_list(),
headers=headers,
timeout=timeout,
) as response:
yield response

6
scrapy/extensions/memusage.py
View File

@ -113,7 +113,7 @@ class MemoryUsage:
{"memusage": mem},
extra={"crawler": self.crawler},
)
if self.notify_mails:
if self.notify_mails: # pragma: no cover
subj = (
f"{self.crawler.settings['BOT_NAME']} terminated: "
f"memory usage exceeded {mem}MiB at {socket.gethostname()}"
@ -146,7 +146,7 @@ class MemoryUsage:
{"memusage": mem},
extra={"crawler": self.crawler},
)
if self.notify_mails:
if self.notify_mails: # pragma: no cover
subj = (
f"{self.crawler.settings['BOT_NAME']} warning: "
f"memory usage reached {mem}MiB at {socket.gethostname()}"
@ -155,7 +155,7 @@ class MemoryUsage:
self.crawler.stats.set_value("memusage/warning_notified", 1)
self.warned = True
def _send_report(self, rcpts: list[str], subject: str) -> None:
def _send_report(self, rcpts: list[str], subject: str) -> None: # pragma: no cover
"""send notification mail with some additional useful info"""
assert self.crawler.engine
assert self.crawler.stats

25
scrapy/pipelines/images.py
View File

@ -11,14 +11,20 @@ import hashlib
import warnings
from contextlib import suppress
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar
from typing import TYPE_CHECKING, Any, ClassVar, cast
from itemadapter import ItemAdapter
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum
from scrapy.pipelines.files import (
FileException,
FilesPipeline,
GCSFilesStore,
S3FilesStore,
_md5sum,
)
from scrapy.utils.defer import ensure_awaitable
from scrapy.utils.python import to_bytes
@ -33,6 +39,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
from scrapy.settings import BaseSettings
class ImageException(FileException):
@ -126,6 +133,20 @@ class ImagesPipeline(FilesPipeline):
) -> str:
return await self.image_downloaded(response, request, info, item=item)
@classmethod
def _update_stores(cls, settings: BaseSettings) -> None:
super()._update_stores(settings)
s3store: type[S3FilesStore] = cast(
"type[S3FilesStore]", cls.STORE_SCHEMES["s3"]
)
s3store.POLICY = settings["IMAGES_STORE_S3_ACL"]
gcs_store: type[GCSFilesStore] = cast(
"type[GCSFilesStore]", cls.STORE_SCHEMES["gs"]
)
gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None
async def image_downloaded(
self,
response: Response,

11
scrapy/utils/misc.py
View File

@ -135,14 +135,9 @@ def walk_modules(path: str) -> list[ModuleType]: # pragma: no cover
return list(walk_modules_iter(path))
def md5sum(file: IO[bytes]) -> str:
def md5sum(file: IO[bytes]) -> str: # pragma: no cover
"""Calculate the md5 checksum of a file-like object without reading its
whole content in memory.
>>> from io import BytesIO
>>> md5sum(BytesIO(b'file content to hash'))
'784406af91dd5a54fbb9c84c2236595a'
"""
whole content in memory."""
warnings.warn(
(
"The scrapy.utils.misc.md5sum function is deprecated and will be "
@ -162,7 +157,7 @@ def md5sum(file: IO[bytes]) -> str:
def rel_has_nofollow(rel: str | None) -> bool:
"""Return True if link rel attribute has nofollow type"""
return rel is not None and "nofollow" in rel.replace(",", " ").split()
return rel is not None and "nofollow" in rel.lower().replace(",", " ").split()
class SupportsFromCrawler(Protocol[_T_co, _P]):

2
tests/ignores.txt
View File

@ -1 +1,3 @@
scrapy/core/downloader/handlers/http.py
scrapy/extensions/statsmailer.py
scrapy/mail.py

45
tests/test_commands.py
View File

@ -12,6 +12,7 @@ import pytest
import scrapy
from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
from scrapy.utils.reactor import _asyncio_reactor_path
from tests.utils.cmdline import call, proc
@ -28,6 +29,50 @@ class EmptyCommand(ScrapyCommand):
pass
class TestHelpDeprecation:
def test_calling_help_is_deprecated(self) -> None:
command = EmptyCommand()
with pytest.warns(
ScrapyDeprecationWarning,
match=r"ScrapyCommand\.help\(\) is deprecated, use long_desc\(\) instead\.",
):
result = command.help()
# help() still delegates to long_desc() for backward compatibility.
assert result == command.long_desc()
def test_overriding_help_is_deprecated(self) -> None:
class HelpCommand(ScrapyCommand):
def short_desc(self) -> str:
return ""
def run(self, args: list[str], opts: argparse.Namespace) -> None:
pass
def help(self) -> str:
return "custom help"
with pytest.warns(
ScrapyDeprecationWarning,
match=r"The ScrapyCommand\.help\(\) method is deprecated and "
r"overriding it, as the .*HelpCommand class does, has no effect; "
r"override long_desc\(\) instead\.",
):
HelpCommand()
def test_not_overriding_help_does_not_warn(self, recwarn) -> None:
# Commands that do not override help() must not emit the
# override-deprecation warning when instantiated, including subclasses
# several levels below ScrapyCommand (as the built-in commands are).
class SubCommand(EmptyCommand):
pass
EmptyCommand()
SubCommand()
assert not [
w for w in recwarn.list if issubclass(w.category, ScrapyDeprecationWarning)
]
class TestCommandSettings:
def setup_method(self):
self.command = EmptyCommand()

3
tests/test_crawl.py
View File

@ -715,6 +715,9 @@ class TestCrawlSpider:
assert isinstance(crawler.spider, SingleRequestSpider)
assert crawler.spider.meta["responses"][0].certificate is None
@pytest.mark.filterwarnings(
r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"
)
@pytest.mark.parametrize(
"url",
[

155
tests/test_downloader_handlers_http_base.py
View File

@ -1148,6 +1148,9 @@ class TestHttpWithCrawlerBase(ABC):
reason = crawler.spider.meta["close_reason"] # type: ignore[attr-defined]
assert reason == "finished"
@pytest.mark.filterwarnings(
r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"
)
@coroutine_test
async def test_response_ssl_certificate(self, mockserver: MockServer) -> None:
if not self.is_secure:
@ -1328,6 +1331,9 @@ class TestHttpProxyBase(ABC):
assert response.body == self.expected_http_proxy_request_body
PROXY_KINDS = ["http", "https", "socks5"]
class TestMitmProxyBase(ABC):
# whether the handler supports HTTPS proxies with HTTPS destinations
handler_supports_tls_in_tls: bool = True
@ -1338,57 +1344,54 @@ class TestMitmProxyBase(ABC):
def settings_dict(self) -> dict[str, Any] | None:
raise NotImplementedError
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@pytest.mark.usefixtures("mitm_proxy_server")
@coroutine_test
async def test_http_proxy(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
) -> None:
"""HTTP proxy, HTTP or HTTPS destination."""
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async(
seed=mockserver.url("/status?n=200", is_secure=https_dest)
)
assert isinstance(crawler.spider, SingleRequestSpider)
self._assert_got_response_code(200, caplog.text)
self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@pytest.mark.usefixtures("mitm_proxy_server_https")
@coroutine_test
async def test_https_proxy(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
) -> None:
"""HTTPS proxy, HTTP or HTTPS destination."""
if https_dest and not self.handler_supports_tls_in_tls:
def _maybe_skip(self, proxy_kind: str, https_dest: bool) -> None:
if proxy_kind == "socks5" and not self.handler_supports_socks:
pytest.skip("SOCKS proxies are not supported")
if (
proxy_kind == "https"
and https_dest
and not self.handler_supports_tls_in_tls
):
pytest.skip("HTTPS proxies for HTTPS destinations are not supported")
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async(
seed=mockserver.url("/status?n=200", is_secure=https_dest)
)
assert isinstance(crawler.spider, SingleRequestSpider)
self._assert_got_response_code(200, caplog.text)
self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
@pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@pytest.mark.usefixtures("mitm_proxy_server")
@coroutine_test
async def test_http_proxy_auth_error(
async def test_proxy(
self,
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
proxy_server: str,
mockserver: MockServer,
https_dest: bool,
) -> None:
"""HTTP proxy, HTTP or HTTPS destination, wrong proxy creds."""
"""HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination."""
self._maybe_skip(proxy_server, https_dest)
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async(
seed=mockserver.url("/status?n=200", is_secure=https_dest)
)
assert isinstance(crawler.spider, SingleRequestSpider)
self._assert_got_response_code(200, caplog.text)
self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
@pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@coroutine_test
async def test_proxy_auth_error(
self,
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
proxy_server: str,
mockserver: MockServer,
https_dest: bool,
) -> None:
"""HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination, wrong proxy creds."""
self._maybe_skip(proxy_server, https_dest)
envvar = "https_proxy" if https_dest else "http_proxy"
monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar]))
crawler = get_crawler(SimpleSpider, self.settings_dict)
@ -1396,20 +1399,28 @@ class TestMitmProxyBase(ABC):
await crawler.crawl_async(
mockserver.url("/status?n=200", is_secure=https_dest)
)
# The proxy returns a 407 error code but it does not reach the client;
# it just sees an exception.
self._assert_got_auth_exception(caplog.text)
if proxy_server == "socks5":
assert "DownloadConnectionRefusedError" in caplog.text
else:
# The proxy returns a 407 error code but it does not reach the
# client; it just sees an exception.
self._assert_got_auth_exception(caplog.text)
@pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@pytest.mark.usefixtures("mitm_proxy_server")
@coroutine_test
async def test_dont_leak_proxy_authorization_header(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
async def test_proxy_dont_leak_auth_header(
self,
caplog: pytest.LogCaptureFixture,
proxy_server: str,
mockserver: MockServer,
https_dest: bool,
) -> None:
"""HTTP proxy, HTTP or HTTPS destination. Check that the auth header
is not sent to the destination."""
"""HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination. Check that the
auth header is not sent to the destination."""
self._maybe_skip(proxy_server, https_dest)
request = Request(mockserver.url("/echo", is_secure=https_dest))
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
@ -1420,48 +1431,36 @@ class TestMitmProxyBase(ABC):
echo = json.loads(crawler.spider.meta["responses"][0].text)
assert "Proxy-Authorization" not in echo["headers"]
@pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@pytest.mark.usefixtures("socks5_proxy_server")
@coroutine_test
async def test_download_with_socks_proxy(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
) -> None:
"""SOCKS5 proxy, HTTP or HTTPS destination."""
if not self.handler_supports_socks:
pytest.skip("SOCKS proxies are not supported")
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async(
seed=mockserver.url("/status?n=200", is_secure=https_dest)
)
assert isinstance(crawler.spider, SingleRequestSpider)
self._assert_got_response_code(200, caplog.text)
self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@pytest.mark.usefixtures("socks5_proxy_server")
@coroutine_test
async def test_socks_proxy_auth_error(
async def test_proxy_redirect(
self,
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
proxy_server: str,
mockserver: MockServer,
https_dest: bool,
) -> None:
if not self.handler_supports_socks:
pytest.skip("SOCKS proxies are not supported")
envvar = "https_proxy" if https_dest else "http_proxy"
monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar]))
crawler = get_crawler(SimpleSpider, self.settings_dict)
"""HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination, following a
redirect. Check that the redirected request still goes through the
proxy and doesn't lose the proxy auth.
"""
self._maybe_skip(proxy_server, https_dest)
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async(
mockserver.url("/status?n=200", is_secure=https_dest)
seed=mockserver.url("/redirect", is_secure=https_dest)
)
assert "DownloadConnectionRefusedError" in caplog.text
assert isinstance(crawler.spider, SingleRequestSpider)
assert crawler.spider.meta.get("failure") is None
responses = crawler.spider.meta.get("responses", [])
assert len(responses) == 1
assert responses[0].status == 200
assert responses[0].url == mockserver.url("/redirected", is_secure=https_dest)
self._assert_got_response_code(200, caplog.text)
self._assert_headers(responses[0].headers, https_dest)
@staticmethod
def _assert_headers(headers: Headers, https_dest: bool) -> None:

115
tests/test_extension_memusage.py Normal file
View File

@ -0,0 +1,115 @@
from __future__ import annotations
import logging
import sys
import pytest
from scrapy import signals
from scrapy.core import engine as engine_mod
from scrapy.exceptions import NotConfigured
from scrapy.extensions import memusage as memusage_mod
from scrapy.extensions.memusage import MemoryUsage
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
from tests.utils import OneShotLoop
from tests.utils.decorators import coroutine_test
# MemoryUsage relies on the stdlib 'resource' module (not available on Windows)
pytestmark = pytest.mark.skipif(
sys.platform.startswith("win"),
reason="MemoryUsage extension not available on Windows",
)
MB = 1024 * 1024
class _LoopSpider(Spider):
name = "loop-data-spider"
def __init__(self, url: str, loops: int = 60, **kw):
super().__init__(**kw)
self.url = url
self.loops = loops
self.start_urls = [url]
def parse(self, response):
count = response.meta.get("count", 0)
if count + 1 < self.loops:
yield response.follow(
self.url, callback=self.parse, meta={"count": count + 1}
)
def test_memusage_disabled() -> None:
settings = {
"MEMUSAGE_ENABLED": False,
}
with pytest.raises(NotConfigured):
MemoryUsage.from_crawler(get_crawler(settings_dict=settings))
@coroutine_test
async def test_memusage_limit_closes_spider_with_reason_and_error_log(
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
) -> None:
settings = {
"MEMUSAGE_LIMIT_MB": 10,
"MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01,
"TELNETCONSOLE_ENABLED": False,
"LOG_LEVEL": "INFO",
}
# Avoid background LoopingCall that can log after the test finishes.
monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop)
# Avoid engine start/stop races (the extension stops the engine in engine_started).
monkeypatch.setattr(engine_mod, "create_looping_call", OneShotLoop)
monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda _: 250 * MB)
crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings)
with caplog.at_level(logging.ERROR, logger="scrapy.extensions.memusage"):
await crawler.crawl_async(url="data:,", loops=100)
assert crawler.stats
assert crawler.stats.get_value("memusage/limit_reached") == 1
assert crawler.stats.get_value("finish_reason") == "memusage_exceeded"
assert any(
"memory usage exceeded" in r.getMessage().lower() for r in caplog.records
)
@coroutine_test
async def test_memusage_warning_logs_but_allows_normal_finish(
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
) -> None:
settings = {
"MEMUSAGE_WARNING_MB": 50,
"MEMUSAGE_LIMIT_MB": 0, # no hard limit
"MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01,
"TELNETCONSOLE_ENABLED": False,
"LOG_LEVEL": "INFO",
}
# Avoid background LoopingCall that can log after the test finishes.
monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop)
monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB)
crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings)
warning_signals: list[int] = []
def on_warning_reached() -> None:
warning_signals.append(1)
crawler.signals.connect(on_warning_reached, signal=signals.memusage_warning_reached)
with caplog.at_level(logging.WARNING, logger="scrapy.extensions.memusage"):
await crawler.crawl_async(url="data:,", loops=60)
assert warning_signals == [1]
assert crawler.stats
assert crawler.stats.get_value("memusage/warning_reached") == 1
assert crawler.stats.get_value("finish_reason") == "finished"
assert any("memory usage reached" in r.getMessage().lower() for r in caplog.records)

23
tests/test_extension_statsmailer.py
View File

@ -1,20 +1,27 @@
import warnings
from unittest.mock import MagicMock
import pytest
from scrapy import signals
from scrapy.exceptions import NotConfigured
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.signalmanager import SignalManager
from scrapy.statscollectors import StatsCollector
from scrapy.utils.spider import DefaultSpider
pytestmark = pytest.mark.filterwarnings(
"ignore:The scrapy.extensions.statsmailer module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning",
"ignore:The scrapy.mail module is deprecated:scrapy.exceptions.ScrapyDeprecationWarning",
)
from scrapy.extensions import statsmailer # noqa: E402
from scrapy.mail import MailSender # noqa: E402
with warnings.catch_warnings():
warnings.filterwarnings(
"ignore",
r"The scrapy\.extensions\.statsmailer module is deprecated",
ScrapyDeprecationWarning,
)
warnings.filterwarnings(
"ignore",
r"The scrapy\.mail module is deprecated",
ScrapyDeprecationWarning,
)
from scrapy.extensions import statsmailer
from scrapy.mail import MailSender
@pytest.fixture

3
tests/test_http2_client_protocol.py
View File

@ -669,6 +669,9 @@ class TestHttps2ClientProtocol:
response = await make_request(client, request)
assert response.status == status
@pytest.mark.filterwarnings(
r"ignore:.*You should use cryptography's X\.509 APIs:DeprecationWarning"
)
@deferred_f_from_coro_f
async def test_response_has_correct_certificate_ip_address(
self,

39
tests/test_pipeline_images.py
View File

@ -14,6 +14,7 @@ from itemadapter import ItemAdapter
from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.pipelines.files import GCSFilesStore, S3FilesStore
from scrapy.pipelines.images import ImageException, ImagesPipeline
from scrapy.utils.test import get_crawler
@ -541,6 +542,44 @@ class TestImagesPipelineCustomSettings:
expected_value = settings.get(settings_attr)
assert getattr(pipeline_cls, pipe_attr.lower()) == expected_value
def test_images_store_s3_acl_setting_used(self, tmp_path):
old_policy = S3FilesStore.POLICY
try:
crawler = get_crawler(
None,
{
"IMAGES_STORE": tmp_path,
"IMAGES_STORE_S3_ACL": "public-read",
"FILES_STORE_S3_ACL": "private",
},
)
ImagesPipeline.from_crawler(crawler)
assert S3FilesStore.POLICY == "public-read"
finally:
S3FilesStore.POLICY = old_policy
def test_images_store_gcs_acl_setting_used(self, tmp_path):
old_policy = GCSFilesStore.POLICY
try:
crawler = get_crawler(
None,
{
"IMAGES_STORE": tmp_path,
"IMAGES_STORE_GCS_ACL": "authenticatedRead",
"FILES_STORE_GCS_ACL": "",
},
)
ImagesPipeline.from_crawler(crawler)
assert GCSFilesStore.POLICY == "authenticatedRead"
finally:
GCSFilesStore.POLICY = old_policy
def _create_image(format_, *a, **kw):
buf = io.BytesIO()

21
tests/test_signalmanager.py Normal file
View File

@ -0,0 +1,21 @@
from scrapy.signalmanager import SignalManager
class TestSignalManager:
def test_disconnect_all(self):
signal = object()
sender = object()
sm = SignalManager(sender)
calls = []
def handler():
calls.append(1)
sm.connect(handler, signal)
sm.send_catch_log(signal)
assert calls == [1]
sm.disconnect_all(signal)
sm.send_catch_log(signal)
assert calls == [1] # handler no longer called after disconnect_all

5
tests/test_spider_sitemap.py
View File

@ -2,7 +2,6 @@ from __future__ import annotations
import gzip
import re
import warnings
from datetime import datetime
from io import BytesIO
from logging import WARNING
@ -429,9 +428,7 @@ Sitemap: /sitemap-relative-url.xml
crawler = get_crawler(TestSpider)
spider = TestSpider.from_crawler(crawler)
with warnings.catch_warnings():
warnings.simplefilter("error")
requests = [request async for request in spider.start()]
requests = [request async for request in spider.start()]
assert len(requests) == 1
request = requests[0]

13
tests/test_spider_start.py
View File

@ -1,6 +1,5 @@
from __future__ import annotations
import warnings
from asyncio import sleep
from typing import Any
@ -45,9 +44,7 @@ class TestMain:
async def parse(self, response):
yield ITEM_A
with warnings.catch_warnings():
warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A])
await self._test_spider(TestSpider, [ITEM_A])
@coroutine_test
async def test_start(self):
@ -57,9 +54,7 @@ class TestMain:
async def start(self):
yield ITEM_A
with warnings.catch_warnings():
warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A])
await self._test_spider(TestSpider, [ITEM_A])
@coroutine_test
async def test_start_subclass(self):
@ -70,9 +65,7 @@ class TestMain:
class TestSpider(BaseSpider):
name = "test"
with warnings.catch_warnings():
warnings.simplefilter("error")
await self._test_spider(TestSpider, [ITEM_A])
await self._test_spider(TestSpider, [ITEM_A])
async def _test_start(self, start_, expected_items=None):
class TestSpider(Spider):

5
tests/test_spidermiddleware_process_start.py
View File

@ -1,4 +1,3 @@
import warnings
from asyncio import sleep
import pytest
@ -76,9 +75,7 @@ class TestMain:
@coroutine_test
async def test_modern_mw_modern_spider(self):
with warnings.catch_warnings():
warnings.simplefilter("error")
await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
async def _test_sleep(self, spider_middlewares):
class TestSpider(Spider):

5
tests/test_utils_misc/__init__.py
View File

@ -160,3 +160,8 @@ class TestUtilsMisc:
assert rel_has_nofollow("nofollowfoo") is False
assert rel_has_nofollow("foonofollow") is False
assert rel_has_nofollow("ugc, , nofollow") is True
# rel attribute values are ASCII case-insensitive per the HTML spec
assert rel_has_nofollow("NoFollow") is True
assert rel_has_nofollow("NOFOLLOW") is True
assert rel_has_nofollow("UGC NoFollow") is True
assert rel_has_nofollow("ugc,NoFollow") is True

8
tests/test_utils_project.py
View File

@ -1,5 +1,4 @@
import os
import warnings
from pathlib import Path
import pytest
@ -41,11 +40,8 @@ class TestGetProjectSettings:
envvars = {
"SCRAPY_SETTINGS_MODULE": value,
}
with warnings.catch_warnings():
warnings.simplefilter("error")
with set_environ(**envvars):
settings = get_project_settings()
with set_environ(**envvars):
settings = get_project_settings()
assert settings.get("SETTINGS_MODULE") == value
def test_invalid_envvar(self):

17
tests/utils/__init__.py
View File

@ -1,5 +1,6 @@
import asyncio
import os
from collections.abc import Callable
from pathlib import Path
from twisted.internet.defer import Deferred
@ -31,3 +32,19 @@ def get_script_run_env() -> dict[str, str]:
env = os.environ.copy()
env["PYTHONPATH"] = pythonpath
return env
class OneShotLoop:
"""Test stub for create_looping_call: run once immediately, no background task."""
def __init__(self, func: Callable[[], None]):
self.func = func
self.running = False
def start(self, _interval: float, now: bool = True) -> None:
self.running = True
if now:
self.func()
def stop(self) -> None:
self.running = False