RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge 3aab6cf7f2 into e28e56aa61

This commit is contained in:
Adrian 2026-08-15 21:15:21 +05:00 committed by GitHub
parent e28e56aa61 3aab6cf7f2
commit 3c6d0e9226
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
41 changed files with 1008 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pyproject.toml
View File

@ -8,7 +8,7 @@ dynamic = ["version"]
description = "A high-level Web Crawling and Web Scraping framework"
dependencies = [
"Twisted>=21.7.0",
"cryptography>=37.0.0",
"cryptography>=41.0.5",
"cssselect>=0.9.1",
"defusedxml>=0.7.1",
"itemadapter>=0.1.0",
@ -17,9 +17,9 @@ dependencies = [
"packaging",
"parsel>=1.5.0",
"protego>=0.1.15",
"pyOpenSSL>=22.0.0",
"pyOpenSSL>=24.3.0",
"queuelib>=1.6.1",
"service_identity>=23.1.0",
"service_identity>=24.2.0",
"tldextract",
"w3lib>=1.17.0",
"zope.interface>=5.1.0",

8
scrapy/utils/_compression.py
View File

@ -44,17 +44,15 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes:
_check_max_size(decompressed_size, max_size)
output_stream = BytesIO()
output_stream.write(first_chunk)
while decompressor.unconsumed_tail:
# Anything left in unconsumed_tail once the stream has ended is not part of
# it, and feeding it back would neither consume it nor produce output.
while decompressor.unconsumed_tail and not decompressor.eof:
output_chunk = decompressor.decompress(
decompressor.unconsumed_tail, max_length=_CHUNK_SIZE
)
decompressed_size += len(output_chunk)
_check_max_size(decompressed_size, max_size)
output_stream.write(output_chunk)
if tail := decompressor.flush():
decompressed_size += len(tail)
_check_max_size(decompressed_size, max_size)
output_stream.write(tail)
return output_stream.getvalue()

2
scrapy/utils/_deps_compat.py
View File

@ -18,8 +18,6 @@ TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0)
PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)
# pyOpenSSL X.509 APIs are deprecated and cryptography-based ones are preferred
PYOPENSSL_X509_DEPRECATED = PYOPENSSL_VERSION >= Version("24.3.0")
# SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable
PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0")

4
scrapy/utils/benchserver.py
View File

@ -33,7 +33,9 @@ def _getarg(
return type_(request.args[name][0]) if name in request.args else default
if __name__ == "__main__":
# The bench command kills this process, so coverage data is never written for
# the lines below.
if __name__ == "__main__": # pragma: no cover
from twisted.internet import reactor
root = Root() # type: ignore[no-untyped-call]

7
scrapy/utils/ossignal.py
View File

@ -10,12 +10,7 @@ SignalHandlerT: TypeAlias = (
Callable[[int, FrameType | None], Any] | int | signal.Handlers | None
)
signal_names: dict[int, str] = {}
for signame in dir(signal):
if signame.startswith("SIG") and not signame.startswith("SIG_"):
signum = getattr(signal, signame)
if isinstance(signum, int):
signal_names[signum] = signame
signal_names: dict[int, str] = {member.value: member.name for member in signal.Signals}
def install_shutdown_handlers(

24
scrapy/utils/python.py
View File

@ -101,7 +101,9 @@ def to_bytes(
return text.encode(encoding, errors)
def _chunk_iter(text: str, chunk_size: int) -> Iterable[tuple[str, int]]:
def _chunk_iter(
text: str, chunk_size: int
) -> Iterable[tuple[str, int]]: # pragma: no cover
offset = len(text)
while True:
offset -= chunk_size * 1024
@ -213,17 +215,15 @@ def get_func_args_dict(
except ValueError:
return {}
if isinstance(func, partial):
partial_args = func.args
partial_kw = func.keywords
args = {}
for name, param in sig.parameters.items():
if name in partial_args:
continue
if partial_kw and name in partial_kw:
continue
args[name] = param
if isinstance(func, partial) and func.keywords:
# The signature of a partial already omits the parameters bound to
# positional arguments, but it keeps those bound to keyword arguments,
# turned into keyword-only parameters with a default.
args = {
name: param
for name, param in sig.parameters.items()
if name not in func.keywords
}
else:
args = sig.parameters

12
scrapy/utils/reactor.py
View File

@ -27,7 +27,7 @@ _T = TypeVar("_T")
_P = ParamSpec("_P")
def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port: # type: ignore[return] # noqa: RET503
def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:
"""Like reactor.listenTCP but tries different ports in a range."""
from twisted.internet import reactor
@ -37,14 +37,14 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:
raise ValueError(f"invalid portrange: {portrange}")
if not portrange:
return reactor.listenTCP(0, factory, interface=host) # type: ignore[no-any-return]
if len(portrange) == 1:
return reactor.listenTCP(portrange[0], factory, interface=host) # type: ignore[no-any-return]
for x in range(portrange[0], portrange[1] + 1):
for x in range(portrange[0], portrange[-1]):
try:
return reactor.listenTCP(x, factory, interface=host) # type: ignore[no-any-return]
except error.CannotListenError:
if x == portrange[1]:
raise
pass
# The last port of the range is tried outside the loop so that its
# CannotListenError propagates to the caller.
return reactor.listenTCP(portrange[-1], factory, interface=host) # type: ignore[no-any-return]
class CallLaterOnce(Generic[_T]):

76
scrapy/utils/ssl.py
View File

@ -11,10 +11,7 @@ import OpenSSL.version
from twisted.internet.ssl import CertificateOptions, TLSVersion
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils._deps_compat import (
PYOPENSSL_X509_DEPRECATED,
TWISTED_TLS_LIMITS_OFFBY1,
)
from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
@ -125,18 +122,17 @@ def _ffi_buf_to_string(buf: Any) -> str:
return to_unicode(pyOpenSSLutil.ffi.string(buf))
def ffi_buf_to_string(buf: Any) -> str: # pragma: no cover
def ffi_buf_to_string(buf: Any) -> str:
warnings.warn(
"ffi_buf_to_string() is deprecated.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return ffi_buf_to_string(buf)
return _ffi_buf_to_string(buf)
def _x509name_to_string(x509name: X509Name) -> str:
# from OpenSSL.crypto.X509Name.__repr__
# only used on pyOpenSSL < 24.3.0
result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512)
pyOpenSSLutil.lib.X509_NAME_oneline(
x509name._name, result_buffer, len(result_buffer)
@ -144,7 +140,7 @@ def _x509name_to_string(x509name: X509Name) -> str:
return _ffi_buf_to_string(result_buffer)
def x509name_to_string(x509name: X509Name) -> str: # pragma: no cover
def x509name_to_string(x509name: X509Name) -> str:
warnings.warn(
"x509name_to_string() is deprecated.",
ScrapyDeprecationWarning,
@ -153,48 +149,15 @@ def x509name_to_string(x509name: X509Name) -> str: # pragma: no cover
return _x509name_to_string(x509name)
def _get_temp_key_info(ssl_object: Any) -> str | None:
# adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()
if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"):
# removed in cryptography 40.0.0 (required starting from pyOpenSSL 23.1.0)
return None
temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")
if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p):
return None
temp_key = temp_key_p[0]
if temp_key == pyOpenSSLutil.ffi.NULL:
return None
temp_key = pyOpenSSLutil.ffi.gc(temp_key, pyOpenSSLutil.lib.EVP_PKEY_free)
key_info = []
key_type = pyOpenSSLutil.lib.EVP_PKEY_id(temp_key)
if key_type == pyOpenSSLutil.lib.EVP_PKEY_RSA:
key_info.append("RSA")
elif key_type == pyOpenSSLutil.lib.EVP_PKEY_DH:
key_info.append("DH")
elif key_type == pyOpenSSLutil.lib.EVP_PKEY_EC:
key_info.append("ECDH")
ec_key = pyOpenSSLutil.lib.EVP_PKEY_get1_EC_KEY(temp_key)
ec_key = pyOpenSSLutil.ffi.gc(ec_key, pyOpenSSLutil.lib.EC_KEY_free)
nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name(
pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key)
)
cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid)
if cname == pyOpenSSLutil.ffi.NULL:
cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid)
key_info.append(_ffi_buf_to_string(cname))
else:
key_info.append(_ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
key_info.append(f"{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits")
return ", ".join(key_info)
def get_temp_key_info(ssl_object: Any) -> str | None: # pragma: no cover
def get_temp_key_info(ssl_object: Any) -> str | None:
# A no-op: it read the negotiated ephemeral key through a binding that
# cryptography 40.0.0 removed.
warnings.warn(
"get_temp_key_info() is deprecated. It's also a no-op with cryptography 40.0.0+.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return _get_temp_key_info(ssl_object)
return None
def get_openssl_version() -> str:
@ -210,23 +173,12 @@ def _log_ssl_conn_debug_info(hostname: str, connection: OpenSSL.SSL.Connection)
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
if PYOPENSSL_X509_DEPRECATED:
if server_cert := connection.get_peer_certificate(as_cryptography=True):
logger.debug(
'SSL connection certificate: issuer "%s", subject "%s"',
server_cert.issuer.rfc4514_string(),
server_cert.subject.rfc4514_string(),
)
else: # noqa: PLR5501
if server_cert_pyopenssl := connection.get_peer_certificate():
logger.debug(
'SSL connection certificate: issuer "%s", subject "%s"',
_x509name_to_string(server_cert_pyopenssl.get_issuer()),
_x509name_to_string(server_cert_pyopenssl.get_subject()),
)
key_info = _get_temp_key_info(connection._ssl)
if key_info:
logger.debug("SSL temp key: %s", key_info)
if server_cert := connection.get_peer_certificate(as_cryptography=True):
logger.debug(
'SSL connection certificate: issuer "%s", subject "%s"',
server_cert.issuer.rfc4514_string(),
server_cert.subject.rfc4514_string(),
)
# Twisted-specific

10
tests/mockserver/utils.py
View File

@ -7,11 +7,9 @@ from typing import TYPE_CHECKING, cast
from cryptography.hazmat.primitives.serialization import load_pem_private_key
from cryptography.x509 import load_pem_x509_certificate
from OpenSSL import SSL
from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey
from twisted.internet.ssl import CertificateOptions, ContextFactory
from scrapy.core.downloader.tls import _TWISTED_VERSION_MAP
from scrapy.utils._deps_compat import PYOPENSSL_X509_DEPRECATED
from scrapy.utils.python import to_bytes
from scrapy.utils.ssl import _get_cert_options_version_kwargs
@ -37,12 +35,8 @@ def ssl_context_factory(
keyfile_path = Path(__file__).parent.parent / keyfile
certfile_path = Path(__file__).parent.parent / certfile
if PYOPENSSL_X509_DEPRECATED:
cert = load_pem_x509_certificate(certfile_path.read_bytes())
key = load_pem_private_key(keyfile_path.read_bytes(), password=None)
else:
cert = load_certificate(FILETYPE_PEM, certfile_path.read_bytes()) # type: ignore[assignment]
key = load_privatekey(FILETYPE_PEM, keyfile_path.read_bytes()) # type: ignore[assignment]
cert = load_pem_x509_certificate(certfile_path.read_bytes())
key = load_pem_private_key(keyfile_path.read_bytes(), password=None)
tls_min = _TWISTED_VERSION_MAP.get(tls_min_version) if tls_min_version else None
tls_max = _TWISTED_VERSION_MAP.get(tls_max_version) if tls_max_version else None

34
tests/test_command_shell.py
View File

@ -191,6 +191,11 @@ class MySpider(scrapy.Spider):
class TestInteractiveShell:
# Starting an interactive shell involves an interpreter start-up, Scrapy
# imports and shell imports, which on PyPy with coverage enabled can take
# well over 10 seconds.
TIMEOUT = 60
def test_fetch(self, mockserver: MockServer) -> None:
args = (
sys.executable,
@ -201,7 +206,7 @@ class TestInteractiveShell:
env = os.environ.copy()
env["SCRAPY_PYTHON_SHELL"] = "python"
logfile = BytesIO()
p = PopenSpawn(args, env=env, timeout=60)
p = PopenSpawn(args, env=env, timeout=self.TIMEOUT)
p.logfile_read = logfile
p.expect_exact("Available Scrapy objects")
p.sendline(f"fetch('{mockserver.url('/')}')")
@ -235,7 +240,7 @@ class TestInteractiveShell:
def _run_interactive_shell(self, env: dict[str, str]) -> str:
args = (sys.executable, "-m", "scrapy.cmdline", "shell")
logfile = BytesIO()
p = PopenSpawn(args, env=env, timeout=60)
p = PopenSpawn(args, env=env, timeout=self.TIMEOUT)
p.logfile_read = logfile
p.expect_exact("Available Scrapy objects")
p.sendeof()
@ -256,7 +261,7 @@ class TestInteractiveShell:
self._isolate_config(env, config_home)
args = (sys.executable, "-m", "scrapy.cmdline", "shell")
logfile = BytesIO()
p = PopenSpawn(args, env=env, timeout=60)
p = PopenSpawn(args, env=env, timeout=self.TIMEOUT)
p.logfile_read = logfile
p.expect_exact("Available Scrapy objects")
# The standard Python shell never imports IPython, whereas the IPython
@ -276,6 +281,29 @@ class TestInteractiveShell:
self._isolate_config(env, config_home)
assert "Traceback" not in self._run_interactive_shell(env)
@pytest.mark.skipif(
importlib.util.find_spec("IPython") is None, reason="IPython is not installed"
)
def test_shell_ipython(self, tmp_path: Path) -> None:
# Reaching the embedded IPython shell requires selecting it explicitly,
# since ptpython takes precedence when both are installed.
config_home = tmp_path / "config"
config_home.mkdir()
env = os.environ.copy()
self._isolate_config(env, config_home)
env["SCRAPY_PYTHON_SHELL"] = "ipython"
args = (sys.executable, "-m", "scrapy.cmdline", "shell")
logfile = BytesIO()
p = PopenSpawn(args, env=env, timeout=self.TIMEOUT)
p.logfile_read = logfile
p.expect_exact("Available Scrapy objects")
p.sendline("import sys; print('IPYMODULE', 'IPython' in sys.modules)")
p.expect_exact("IPYMODULE True")
p.sendeof()
p.wait() # type: ignore[no-untyped-call]
logfile.seek(0)
assert "Traceback" not in logfile.read().decode()
@pytest.fixture
def restore_sigint():

24
tests/test_crawl.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import json
import logging
import re
from ipaddress import IPv4Address
from socket import gethostbyname
from typing import TYPE_CHECKING, Any
@ -18,7 +19,11 @@ from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownloa
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.engine import format_engine_status, get_engine_status
from scrapy.utils.engine import (
format_engine_status,
get_engine_status,
print_engine_status,
)
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
@ -372,6 +377,23 @@ with multiples lines
assert s["engine.spider.name"] == crawler.spider.name
assert s["len(engine.scraper.slot.active)"] == 1
@coroutine_test
async def test_print_engine_status(
self, mockserver: MockServer, capsys: pytest.CaptureFixture[str]
) -> None:
def cb(response):
assert crawler.engine
print_engine_status(crawler.engine)
crawler = get_crawler(SingleRequestSpider)
await crawler.crawl_async(
seed=mockserver.url("/"), callback_func=cb, mockserver=mockserver
)
assert isinstance(crawler.spider, SingleRequestSpider)
out = capsys.readouterr().out
assert out.startswith("Execution engine status")
assert re.search(rf"engine\.spider\.name +: {crawler.spider.name}\n", out)
@coroutine_test
async def test_format_engine_status(self, mockserver: MockServer) -> None:
est: list[str] = []

8
tests/test_crawler_subprocess.py
View File

@ -19,6 +19,10 @@ from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from tests.mockserver.http import MockServer
# Guards against a hung subprocess. Generous, because starting a script is
# slow on PyPy, slower still with coverage measurement on.
SCRIPT_TIMEOUT = 60
class ScriptRunnerMixin(ABC):
@property
@ -216,7 +220,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
) -> None:
sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK # type: ignore[attr-defined]
args = self.get_script_args(script, "3", *extra_args)
p = PopenSpawn(args, timeout=5, env=get_script_run_env())
p = PopenSpawn(args, timeout=SCRIPT_TIMEOUT, env=get_script_run_env())
p.expect_exact("Spider opened")
p.expect_exact("Crawled (200)")
p.kill(sig)
@ -234,7 +238,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
async def _test_shutdown_forced(self, script: str = "sleeping.py") -> None:
sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK # type: ignore[attr-defined]
args = self.get_script_args(script, "10")
p = PopenSpawn(args, timeout=5, env=get_script_run_env())
p = PopenSpawn(args, timeout=SCRIPT_TIMEOUT, env=get_script_run_env())
p.expect_exact("Spider opened")
p.expect_exact("Crawled (200)")
p.kill(sig)

5
tests/test_downloader_handler_httpx.py
View File

@ -22,6 +22,7 @@ from tests.utils.bases.download_handlers_http import (
TestHttpProxyBase,
TestHttpsBase,
TestHttpsCustomCiphersBase,
TestHttpsDefaultCiphersBase,
TestHttpsInvalidDNSIdBase,
TestHttpsInvalidDNSPatternBase,
TestHttpsTLSVersionBase,
@ -137,6 +138,10 @@ class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBa
pass
class TestHttpsDefaultCiphers(HttpxDownloadHandlerMixin, TestHttpsDefaultCiphersBase):
pass
class TestHttpsTLSVersion(HttpxDownloadHandlerMixin, TestHttpsTLSVersionBase):
pass

53
tests/test_downloader_handlers.py
View File

@ -9,15 +9,31 @@ from tempfile import mkdtemp, mkstemp
from unittest import mock
import pytest
from twisted.internet.defer import CancelledError
from twisted.internet.error import ConnectionRefusedError as TxConnectionRefusedError
from twisted.internet.error import DNSLookupError
from twisted.internet.error import TimeoutError as TxTimeoutError
from twisted.web.client import ResponseFailed
from twisted.web.error import SchemeNotSupported
from w3lib.url import path_to_file_uri
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exceptions import (
CannotResolveHostError,
DownloadCancelledError,
DownloadConnectionRefusedError,
DownloadFailedError,
DownloadTimeoutError,
NotConfigured,
ScrapyDeprecationWarning,
UnsupportedURLSchemeError,
)
from scrapy.http import Request, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils._download_handlers import wrap_twisted_exceptions
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
@ -407,3 +423,38 @@ class TestDataURI:
request = Request("data:,")
response = await self.download_request(request)
assert response.protocol is None
class TestWrapTwistedExceptions:
"""Every Twisted exception the HTTP handlers can raise has a Scrapy equivalent.
Most mappings are also covered end to end by the HTTP handler tests, but a
Twisted ``TimeoutError`` needs a connection attempt that is silently dropped
rather than refused, which no mock server can offer reliably.
"""
@pytest.mark.parametrize(
("twisted_exception", "scrapy_exception"),
[
(SchemeNotSupported, UnsupportedURLSchemeError),
(CancelledError, DownloadCancelledError),
(TxConnectionRefusedError, DownloadConnectionRefusedError),
(DNSLookupError, CannotResolveHostError),
(ResponseFailed, DownloadFailedError),
(TxTimeoutError, DownloadTimeoutError),
],
)
def test_mapping(
self,
twisted_exception: type[Exception],
scrapy_exception: type[Exception],
) -> None:
original = twisted_exception("some message")
with pytest.raises(scrapy_exception) as exc_info, wrap_twisted_exceptions():
raise original
assert exc_info.value.__cause__ is original
assert str(exc_info.value) == str(original)
def test_other_exceptions_pass_through(self) -> None:
with pytest.raises(ZeroDivisionError), wrap_twisted_exceptions():
1 / 0

59
tests/test_downloadermiddleware_httpcompression.py
View File

@ -1,8 +1,10 @@
import zlib
from gzip import GzipFile
from importlib.util import find_spec
from io import BytesIO
from logging import WARNING
from pathlib import Path
from threading import Thread
from typing import Any
import pytest
@ -16,7 +18,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWar
from scrapy.http import HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils._compression import _DecompressionMaxSizeExceeded
from scrapy.utils._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded
from scrapy.utils.gz import gunzip
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
@ -495,6 +497,57 @@ class TestHttpCompression:
self.assertStatsEqual("httpcompression/response_count", None)
self.assertStatsEqual("httpcompression/response_bytes", None)
@pytest.mark.parametrize(
"extra",
[
pytest.param(b"garbage", id="trailing-garbage"),
pytest.param(zlib.compress(b"b" * 100_000), id="second-zlib-stream"),
],
)
def test_deflate_extra_data_after_stream(self, extra: bytes) -> None:
"""Bytes after the end of a deflate stream must not stall decompression.
Such bytes stay in ``unconsumed_tail`` and produce no output, so feeding
them back to the decompressor never makes progress. Decompressing in a
thread lets the test fail instead of hanging if that ever regresses.
"""
# The body must decompress to more than _CHUNK_SIZE, or the whole stream
# is consumed by the first decompress() call and no loop is entered.
plain = b"a" * 100_000
response = Response(
"http://example.com",
body=zlib.compress(plain) + extra,
headers={"Content-Encoding": "deflate"},
)
result: list[Request | Response] = []
thread = Thread(
target=lambda: result.append(
self.mw.process_response(Request("http://example.com"), response)
),
daemon=True,
)
thread.start()
thread.join(60)
assert not thread.is_alive(), "decompression did not terminate"
assert result[0].body == plain
def test_deflate_chunked_output_leaves_nothing_to_flush(self) -> None:
"""Guard the zlib behaviour that lets ``_inflate()`` skip ``flush()``.
``decompress(max_length=...)`` leaves the input it could not turn into
output in ``unconsumed_tail``, so once that is empty every complete
stream has been fully emitted. If a future zlib buffers output instead,
this fails and ``_inflate()`` needs to flush the remainder again.
"""
for size in (_CHUNK_SIZE - 1, _CHUNK_SIZE, _CHUNK_SIZE + 1, 4 * _CHUNK_SIZE):
decompressor = zlib.decompressobj()
decompressor.decompress(zlib.compress(b"a" * size), max_length=_CHUNK_SIZE)
while decompressor.unconsumed_tail and not decompressor.eof:
decompressor.decompress(
decompressor.unconsumed_tail, max_length=_CHUNK_SIZE
)
assert decompressor.flush() == b"", f"pending output for size {size}"
def _test_compression_bomb_setting(self, compression_id: str) -> None:
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
@ -509,6 +562,10 @@ class TestHttpCompression:
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
assert str(cause) == (
f"The number of bytes decompressed so far ({cause.decompressed_size} B) "
"exceeded the specified maximum (1000000 B)."
)
def test_compression_bomb_setting_br(self):
self._test_compression_bomb_setting("br")

15
tests/test_downloaderslotssettings.py
View File

@ -70,15 +70,14 @@ class TestCrawl:
yield crawler.crawl(mockserver=self.mockserver)
slots = crawler.engine.downloader.slots
times = crawler.spider.times
tolerance = 0.3
# Downloading and processing a response add a roughly constant amount
# of time on top of the configured delay, so the margin is absolute.
# It stays well below the 1 second that separates the delays being
# compared, so a slot using the delay of another one still fails.
tolerance = 0.75
delays_real = {k: v[1] - v[0] for k, v in times.items()}
error_delta = {
k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay)
for k, v in slots.items()
}
assert max(list(error_delta.values())) < tolerance
for slot, (first, second) in times.items():
assert abs((second - first) - slots[slot].delay) < tolerance
@coroutine_test

7
tests/test_feedexport_storages.py
View File

@ -172,6 +172,13 @@ class TestFTPFeedStorage:
await self._store(url, b"bar", settings=settings)
self._assert_stored(ftp_server.path / filename, b"bar")
@coroutine_test
async def test_missing_parent_directories(self):
with MockFTPServer() as ftp_server:
path = "missing/parent/dirs/file"
await self._store(ftp_server.url(path), b"foo")
self._assert_stored(ftp_server.path / path, b"foo")
@coroutine_test
async def test_tls(self, monkeypatch):
monkeypatch.setenv(

13
tests/test_linkextractors.py
View File

@ -908,6 +908,19 @@ class TestLxmlParserLinkExtractor:
Link(url="http://example.com/page.html", text="Link", nofollow=False),
]
def test_deduplicates_by_canonical_url(self):
# With canonicalized=False, the default, URLs are canonicalized to
# decide whether two extracted links are the same one.
html = (
b'<a href="http://example.com/page.html?b=2&amp;a=1">1</a>'
b'<a href="http://example.com/page.html?a=1&amp;b=2">2</a>'
)
response = HtmlResponse("http://example.com/", body=html)
lx = LxmlParserLinkExtractor(unique=True)
assert lx.extract_links(response) == [
Link(url="http://example.com/page.html?b=2&a=1", text="1", nofollow=False),
]
def test_strip_false(self):
# With strip=False, trailing whitespace on a relative href survives urljoin
# and is visible to process_value (safe_url_string cleans it up afterward).

25
tests/test_squeues.py
View File

@ -1,5 +1,6 @@
import pickle
import sys
from typing import Any, cast
import pytest
from queuelib.tests import test_queue as t
@ -13,6 +14,8 @@ from scrapy.squeues import (
_MarshalLifoSerializationDiskQueue,
_PickleFifoSerializationDiskQueue,
_PickleLifoSerializationDiskQueue,
_scrapy_non_serialization_queue,
_serializable_queue,
)
@ -20,6 +23,28 @@ class MyItem(Item):
name = Field()
class NoPeekQueue:
"""Queue class without the optional peek method."""
_no_peek_queue = cast("Any", NoPeekQueue)
@pytest.mark.parametrize(
"queue_class",
[
_serializable_queue(_no_peek_queue, pickle.dumps, pickle.loads),
_scrapy_non_serialization_queue(_no_peek_queue),
],
)
def test_peek_unsupported(queue_class):
with pytest.raises(
NotImplementedError,
match="The underlying queue class does not implement 'peek'",
):
queue_class().peek()
def _test_procesor(x):
return x + x

58
tests/test_squeues_request.py
View File

@ -8,7 +8,6 @@ from abc import ABC, abstractmethod
from typing import TYPE_CHECKING
import pytest
import queuelib
from scrapy.http import Request
from scrapy.spiders import Spider
@ -24,12 +23,11 @@ from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
import queuelib
from scrapy.crawler import Crawler
HAVE_PEEK = hasattr(queuelib.queue.FifoMemoryQueue, "peek")
@pytest.fixture
def crawler() -> Crawler:
return get_crawler(Spider)
@ -41,46 +39,26 @@ class TestRequestQueueBase(ABC):
def is_fifo(self) -> bool:
raise NotImplementedError
@pytest.mark.parametrize("test_peek", [True, False])
def test_one_element(self, q: queuelib.queue.BaseQueue, test_peek: bool):
if test_peek and not HAVE_PEEK:
pytest.skip("The queuelib queues do not define peek")
if not test_peek and HAVE_PEEK:
pytest.skip("The queuelib queues define peek")
def test_one_element(self, q: queuelib.queue.BaseQueue):
assert len(q) == 0
if test_peek:
assert q.peek() is None
assert q.peek() is None
assert q.pop() is None
req = Request("http://www.example.com")
q.push(req)
assert len(q) == 1
if test_peek:
result = q.peek()
assert result is not None
assert result.url == req.url
else:
with pytest.raises(
NotImplementedError,
match="The underlying queue class does not implement 'peek'",
):
q.peek()
result = q.peek()
assert result is not None
assert result.url == req.url
result = q.pop()
assert result is not None
assert result.url == req.url
assert len(q) == 0
if test_peek:
assert q.peek() is None
assert q.peek() is None
assert q.pop() is None
@pytest.mark.parametrize("test_peek", [True, False])
def test_order(self, q: queuelib.queue.BaseQueue, test_peek: bool):
if test_peek and not HAVE_PEEK:
pytest.skip("The queuelib queues do not define peek")
if not test_peek and HAVE_PEEK:
pytest.skip("The queuelib queues define peek")
def test_order(self, q: queuelib.queue.BaseQueue):
assert len(q) == 0
if test_peek:
assert q.peek() is None
assert q.peek() is None
assert q.pop() is None
req1 = Request("http://www.example.com/1")
req2 = Request("http://www.example.com/2")
@ -88,25 +66,17 @@ class TestRequestQueueBase(ABC):
q.push(req1)
q.push(req2)
q.push(req3)
if not test_peek:
with pytest.raises(
NotImplementedError,
match="The underlying queue class does not implement 'peek'",
):
q.peek()
reqs = [req1, req2, req3] if self.is_fifo else [req3, req2, req1]
for i, req in enumerate(reqs):
assert len(q) == 3 - i
if test_peek:
result = q.peek()
assert result is not None
assert result.url == req.url
result = q.peek()
assert result is not None
assert result.url == req.url
result = q.pop()
assert result is not None
assert result.url == req.url
assert len(q) == 0
if test_peek:
assert q.peek() is None
assert q.peek() is None
assert q.pop() is None

40
tests/test_utils_benchserver.py Normal file
View File

@ -0,0 +1,40 @@
from __future__ import annotations
import re
from typing import TYPE_CHECKING, cast
from scrapy.utils.benchserver import Root, _getarg
if TYPE_CHECKING:
from twisted.web.server import Request
class _Request:
def __init__(self, **args: bytes) -> None:
self.args = {name.encode(): [value] for name, value in args.items()}
self.written: list[bytes] = []
def write(self, data: bytes) -> None:
self.written.append(data)
def test_getarg() -> None:
request = cast("Request", _Request(total=b"5"))
assert _getarg(request, b"total", 100, int) == 5
assert _getarg(request, b"show", 100, int) == 100
assert _getarg(request, b"missing") is None
def test_render() -> None:
root = Root() # type: ignore[no-untyped-call]
request = _Request(total=b"5", show=b"2")
assert root.getChild("follow", cast("Request", request)) is root
assert root.render(cast("Request", request)) == b""
body = b"".join(request.written).decode()
assert body.startswith("<html><head></head><body>")
assert body.endswith("</body></html>")
numbers = re.findall(
r"<a href='/follow\?total=5&show=2&n=(\d+)'>follow \1</a>", body
)
assert len(numbers) == 2
assert all(1 <= int(number) <= 5 for number in numbers)

20
tests/test_utils_conf.py
View File

@ -9,8 +9,10 @@ from scrapy.settings import BaseSettings, Settings
from scrapy.utils.conf import (
arglist_to_dict,
build_component_list,
closest_scrapy_cfg,
feed_complete_default_values_from_settings,
feed_process_params_from_cli,
get_sources,
)
@ -39,6 +41,20 @@ class TestBuildComponentList:
):
build_component_list(duplicate_bs, convert=lambda x: x.lower())
def test_duplicate_components_in_dict(self):
d = {"one": 1, "ONE": 2}
with pytest.raises(
ValueError, match=r"Some paths in .* convert to the same object"
):
build_component_list(d, convert=lambda x: x.lower())
def test_invalid_value(self):
d = {"one": "1"}
with pytest.raises(
ValueError, match=r"Invalid value 1 for component one, please provide"
):
build_component_list(d, convert=lambda x: x)
def test_valid_numbers(self):
# work well with None and numeric values
d = {"a": 10, "b": None, "c": 15, "d": 5.0}
@ -51,6 +67,10 @@ class TestBuildComponentList:
assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"]
def test_get_sources():
assert get_sources() == [*get_sources(use_closest=False), closest_scrapy_cfg()]
def test_arglist_to_dict():
assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == {
"arg1": "val1",

29
tests/test_utils_console.py
View File

@ -4,6 +4,7 @@ import subprocess
import sys
from importlib.util import find_spec
from io import BytesIO
from types import ModuleType
from typing import TYPE_CHECKING
import pytest
@ -44,6 +45,15 @@ def test_get_shell_embed_func():
assert shell.__name__ == "_embed_standard_shell"
def test_get_shell_embed_func_known_shells():
def embed(namespace: dict[str, object] | None = None, banner: str = "") -> None:
pass
known_shells = {"custom": lambda: embed}
assert get_shell_embed_func(["custom"], known_shells) is embed
assert get_shell_embed_func(["python"], known_shells) is None
def test_get_shell_embed_func_python():
# the standard shell is always available
shell = get_shell_embed_func(["python"])
@ -58,6 +68,25 @@ def test_get_shell_embed_func_bpython():
assert shell.__name__ == "_embed_bpython_shell"
def test_embed_bpython_shell(monkeypatch: pytest.MonkeyPatch) -> None:
# bpython is never the default shell, so a stand-in module takes the place
# of the interactive session the IPython tests below use.
calls: list[tuple[dict[str, object], str]] = []
bpython = ModuleType("bpython")
monkeypatch.setattr(
bpython,
"embed",
lambda locals_, banner: calls.append((locals_, banner)),
raising=False,
)
monkeypatch.setitem(sys.modules, "bpython", bpython)
shell = get_shell_embed_func(["bpython"])
assert shell is not None
shell({"a": 1}, "SHELL-READY")
assert calls == [({"a": 1}, "SHELL-READY")]
def test_get_shell_embed_func_ipython():
pytest.importorskip("IPython")
shell = get_shell_embed_func(["ipython"])

20
tests/test_utils_curl.py
View File

@ -41,6 +41,26 @@ class TestCurlToRequestKwargs:
}
self._test_command(curl_command, expected_result)
def test_get_cookie_option(self):
curl_command = 'curl "http://example.org/" -b "a=1; b=2"'
expected_result = {
"method": "GET",
"url": "http://example.org/",
"cookies": {"a": "1", "b": "2"},
}
self._test_command(curl_command, expected_result)
def test_get_cookie_file_option(self):
# curl reads cookies from a file when the value is not a key-value
# pair; Scrapy ignores it.
curl_command = 'curl "http://example.org/" -b cookies.txt -b "a=1"'
expected_result = {
"method": "GET",
"url": "http://example.org/",
"cookies": {"a": "1"},
}
self._test_command(curl_command, expected_result)
def test_get_complex(self):
curl_command = (
"curl 'http://httpbin.org/get' -H 'Accept-Encoding: gzip, deflate'"

24
tests/test_utils_defer.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import random
import warnings
from asyncio import Future
from typing import TYPE_CHECKING, Any
@ -16,6 +17,7 @@ from scrapy.utils.defer import (
deferred_to_future,
iter_errback,
maybe_deferred_to_future,
maybeDeferred_coro,
mustbe_deferred,
parallel_async,
)
@ -357,6 +359,15 @@ class TestDeferredToFuture:
assert future_result == 42
@pytest.mark.only_not_asyncio
class TestDeferredToFutureNotAsyncio:
def test_deferred(self):
with pytest.raises(
RuntimeError, match=r"deferred_to_future\(\) requires an installed asyncio"
):
deferred_to_future(Deferred())
@pytest.mark.only_asyncio
class TestMaybeDeferredToFutureAsyncio:
@coroutine_test
@ -400,3 +411,16 @@ class TestMaybeDeferredToFutureNotAsyncio:
result = maybe_deferred_to_future(d)
assert isinstance(result, Deferred)
assert result is d
def test_maybe_deferred_coro_deferred() -> None:
d: Deferred[int] = Deferred()
with warnings.catch_warnings(record=True) as records:
warnings.simplefilter("always")
assert maybeDeferred_coro(lambda: d) is d
# Only the deprecation of maybeDeferred_coro() itself is reported; callables
# that return a Deferred are the reason it exists.
assert [str(record.message) for record in records] == [
"maybeDeferred_coro() is deprecated and will be removed in a future"
" Scrapy version."
]

24
tests/test_utils_deprecate.py
View File

@ -7,7 +7,7 @@ from unittest import mock
import pytest
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import create_deprecated_class, update_classpath
from scrapy.utils.deprecate import attribute, create_deprecated_class, update_classpath
class MyWarning(UserWarning):
@ -252,6 +252,11 @@ class TestWarnWhenSubclassed:
):
create_deprecated_class("DeprecatedName", NewName)
def test_unknown_parent_module(self):
with mock.patch("inspect.getmodule", return_value=None):
cls = create_deprecated_class("DeprecatedName", NewName)
assert cls.__module__ == "scrapy.utils.deprecate"
@mock.patch(
"scrapy.utils.deprecate.DEPRECATION_RULES",
@ -284,3 +289,20 @@ class TestUpdateClassPath:
def test_returns_nonstring(self):
for notastring in [None, True, [1, 2, 3], object()]:
assert update_classpath(notastring) == notastring
class TestAttribute:
class MyClass:
pass
def test_default_version(self):
with pytest.warns(
ScrapyDeprecationWarning,
match=r"MyClass\.old attribute is deprecated and will be no longer "
r"supported in Scrapy 0\.12, use MyClass\.new attribute instead",
):
attribute(self.MyClass(), "old", "new")
def test_custom_version(self):
with pytest.warns(ScrapyDeprecationWarning, match="in Scrapy 3.0"):
attribute(self.MyClass(), "old", "new", "3.0")

13
tests/test_utils_display.py
View File

@ -1,11 +1,14 @@
from __future__ import annotations
import builtins
import sys
from io import StringIO
from typing import TYPE_CHECKING
from unittest import mock
from scrapy.utils.display import pformat, pprint
import pytest
from scrapy.utils.display import _enable_windows_terminal_processing, pformat, pprint
if TYPE_CHECKING:
from collections.abc import Mapping, Sequence
@ -81,6 +84,14 @@ def test_pformat_windows(
assert pformat(value) in colorized_strings
@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only console API")
def test_enable_windows_terminal_processing() -> None:
# The tests above mock this out, so this is the only place the real
# kernel32 calls run. Whether they succeed depends on whether stdout is a
# console, which pytest does not guarantee.
assert isinstance(_enable_windows_terminal_processing(), bool)
@mock.patch("sys.platform", "linux")
@mock.patch("sys.stdout.isatty")
def test_pformat_no_pygments(isatty: mock.Mock) -> None:

34
tests/test_utils_iterators.py
View File

@ -210,6 +210,31 @@ class TestXmliter:
with pytest.raises(StopIteration):
next(my_iter)
def test_xmliter_namespaced_nodename_other_prefixes(self):
body = b"""<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0"
xmlns:a="http://example.com/ns/a"
xmlns:g="http://base.google.com/ns/1.0">
<item>
<a:id>A_1</a:id>
<g:id>ITEM_1</g:id>
</item>
</rss>
"""
response = XmlResponse(url="http://example.com", body=body)
node = next(xmliter_lxml(response, "g:id"))
assert node.xpath("text()").getall() == ["ITEM_1"]
def test_xmliter_non_text_response(self):
body = (
b'<?xml version="1.0" encoding="UTF-8"?>'
b"<products><product>one</product></products>"
)
response = Response(url="http://example.com", body=body)
assert [
node.xpath("text()").get() for node in xmliter_lxml(response, "product")
] == ["one"]
def test_xmliter_exception(self):
body = (
'<?xml version="1.0" encoding="UTF-8"?>'
@ -421,6 +446,9 @@ class TestUtilsCsv:
{"id": "4", "name": "empty", "value": ""},
]
def test_csviter_empty(self):
assert not list(csviter(""))
def test_csviter_exception(self):
body = get_testdata("feeds", "feed-sample3.csv")
@ -481,6 +509,12 @@ class TestBodyOrStr:
assert type(r1) is type(r2)
assert type(r1) is not type(r3) # type: ignore[comparison-overlap]
def test_body_or_str_unsupported_type(self) -> None:
with pytest.raises(
TypeError, match="must be Response or str or bytes, not int"
):
_body_or_str(42) # type: ignore[call-overload]
@staticmethod
def _assert_type_and_value(
a: str | bytes, b: str | bytes, obj: Response | str | bytes

24
tests/test_utils_job.py Normal file
View File

@ -0,0 +1,24 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from scrapy.settings import Settings
from scrapy.utils.job import job_dir
if TYPE_CHECKING:
from pathlib import Path
def test_no_jobdir() -> None:
assert job_dir(Settings()) is None
assert job_dir(Settings({"JOBDIR": ""})) is None
def test_existing_jobdir(tmp_path: Path) -> None:
assert job_dir(Settings({"JOBDIR": str(tmp_path)})) == str(tmp_path)
def test_missing_jobdir(tmp_path: Path) -> None:
jobdir = tmp_path / "missing" / "jobdir"
assert job_dir(Settings({"JOBDIR": str(jobdir)})) == str(jobdir)
assert jobdir.is_dir()

70
tests/test_utils_log.py
View File

@ -9,15 +9,21 @@ from io import StringIO
from typing import TYPE_CHECKING, Any, cast
import pytest
from twisted.python import log as twisted_log
from twisted.python.failure import Failure
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
from scrapy.utils.log import (
LogCounterHandler,
SpiderLoggerAdapter,
StreamLogger,
TopLevelFormatter,
_uninstall_scrapy_root_handler,
configure_logging,
failure_to_exc_info,
get_scrapy_root_handler,
install_scrapy_root_handler,
logformatter_adapter,
)
from scrapy.utils.test import get_crawler
@ -122,6 +128,70 @@ class TestStreamLogger:
sys.stdout = old_stdout
def test_flush(self) -> None:
class FlushCountingHandler(logging.Handler):
flushes = 0
def flush(self) -> None:
self.flushes += 1
handler = FlushCountingHandler()
logger = logging.getLogger("test_flush")
logger.addHandler(handler)
try:
StreamLogger(logger).flush()
finally:
logger.removeHandler(handler)
assert handler.flushes == 1
class TestConfigureLogging:
@pytest.fixture(autouse=True)
def restore_logging(self) -> Generator[None]:
root_handlers = logging.root.handlers[:]
# configure_logging() adds a Twisted log observer without giving any way
# to remove it afterwards.
observers = twisted_log.theLogPublisher.observers[:]
old_stdout = sys.stdout
old_showwarning = warnings.showwarning
try:
yield
finally:
warnings.showwarning = old_showwarning
sys.stdout = old_stdout
twisted_log.theLogPublisher.observers[:] = observers
logging.root.handlers[:] = root_handlers
_uninstall_scrapy_root_handler()
@staticmethod
def _warnings_are_captured() -> bool:
return warnings.showwarning.__module__ == "logging"
def test_log_stdout(self) -> None:
configure_logging(settings={"LOG_STDOUT": True}, install_root_handler=False)
assert isinstance(sys.stdout, StreamLogger)
def test_captures_warnings(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(sys, "warnoptions", [])
logging.captureWarnings(False)
configure_logging(install_root_handler=False)
assert self._warnings_are_captured()
def test_keeps_warnoptions(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(sys, "warnoptions", ["default"])
logging.captureWarnings(False)
configure_logging(install_root_handler=False)
assert not self._warnings_are_captured()
def test_reinstall_root_handler_removed_from_root(self) -> None:
install_scrapy_root_handler(Settings())
handler = get_scrapy_root_handler()
assert handler is not None
# Something else removed the handler from the root logger.
logging.root.removeHandler(handler)
install_scrapy_root_handler(Settings())
assert get_scrapy_root_handler() is not handler
@pytest.mark.parametrize(
("base_extra", "log_extra", "expected_extra"),

54
tests/test_utils_ossignal.py Normal file
View File

@ -0,0 +1,54 @@
from __future__ import annotations
import signal
from typing import TYPE_CHECKING, Any
import pytest
from scrapy.utils.ossignal import install_shutdown_handlers
if TYPE_CHECKING:
from collections.abc import Generator
from types import FrameType
SHUTDOWN_SIGNALS = [
getattr(signal, name)
for name in ("SIGTERM", "SIGINT", "SIGBREAK")
if hasattr(signal, name)
]
@pytest.fixture(autouse=True)
def restore_handlers() -> Generator[None]:
handlers = {sig: signal.getsignal(sig) for sig in SHUTDOWN_SIGNALS}
try:
yield
finally:
for sig, handler in handlers.items():
signal.signal(sig, handler)
def shutdown(signum: int, frame: FrameType | None) -> Any:
pass
def test_install() -> None:
install_shutdown_handlers(shutdown)
for sig in SHUTDOWN_SIGNALS:
assert signal.getsignal(sig) is shutdown
def test_keeps_custom_sigint_handler() -> None:
def custom(signum: int, frame: FrameType | None) -> Any:
pass
signal.signal(signal.SIGINT, custom)
install_shutdown_handlers(shutdown, override_sigint=False)
assert signal.getsignal(signal.SIGINT) is custom
assert signal.getsignal(signal.SIGTERM) is shutdown
def test_overrides_default_sigint_handler() -> None:
signal.signal(signal.SIGINT, signal.default_int_handler)
install_shutdown_handlers(shutdown, override_sigint=False)
assert signal.getsignal(signal.SIGINT) is shutdown

66
tests/test_utils_project.py
View File

@ -6,8 +6,14 @@ from typing import TYPE_CHECKING
import pytest
from scrapy.exceptions import NotConfigured
from scrapy.utils.misc import set_environ
from scrapy.utils.project import data_path, get_project_settings
from scrapy.utils.project import (
data_path,
get_project_settings,
inside_project,
project_data_dir,
)
if TYPE_CHECKING:
from collections.abc import Generator
@ -27,6 +33,19 @@ def proj_path(tmp_path: Path) -> Generator[Path]:
os.chdir(prev_dir)
@pytest.fixture
def no_proj_path(tmp_path: Path) -> Generator[Path]:
"""A working directory without a scrapy.cfg file, also isolated from the
user-wide and system-wide Scrapy configuration files."""
prev_dir = Path.cwd()
try:
os.chdir(tmp_path)
with set_environ(HOME=str(tmp_path), XDG_CONFIG_HOME=str(tmp_path)):
yield tmp_path
finally:
os.chdir(prev_dir)
def test_data_path_outside_project() -> None:
assert str(Path(".scrapy", "somepath")) == data_path("somepath")
abspath = str(Path(os.path.sep, "absolute", "path"))
@ -40,6 +59,51 @@ def test_data_path_inside_project(proj_path: Path) -> None:
assert abspath == data_path(abspath)
def test_data_path_createdir(no_proj_path: Path) -> None:
path = Path(data_path("somepath", createdir=True))
assert path.is_dir()
# An existing directory is left alone.
assert Path(data_path("somepath", createdir=True)) == path
def test_inside_project_unimportable_settings_module(no_proj_path: Path) -> None:
with (
set_environ(SCRAPY_SETTINGS_MODULE="tests.no_such_settings_module"),
pytest.warns(
UserWarning, match="Cannot import scrapy settings module tests.no_such"
),
):
assert inside_project() is False
def test_project_data_dir_outside_project(no_proj_path: Path) -> None:
with pytest.raises(NotConfigured, match="Not inside a project"):
project_data_dir()
def test_project_data_dir_without_scrapy_cfg(no_proj_path: Path) -> None:
with (
set_environ(SCRAPY_SETTINGS_MODULE="tests.test_cmdline.settings"),
pytest.raises(NotConfigured, match=r"Unable to find scrapy\.cfg file"),
):
project_data_dir()
def test_project_data_dir_default(proj_path: Path) -> None:
expected = (proj_path / ".scrapy").resolve()
assert Path(project_data_dir()) == expected
assert expected.is_dir()
# A second call finds the directory already created.
assert Path(project_data_dir()) == expected
def test_project_data_dir_from_scrapy_cfg(proj_path: Path) -> None:
datadir = proj_path / "custom-datadir"
Path("scrapy.cfg").write_text(f"[datadir]\ndefault = {datadir}\n")
assert Path(project_data_dir()) == datadir
assert datadir.is_dir()
class TestGetProjectSettings:
def test_valid_envvar(self):
value = "tests.test_cmdline.settings"

39
tests/test_utils_python.py
View File

@ -12,8 +12,10 @@ from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.defer import aiter_errback
from scrapy.utils.python import (
MutableAsyncChain,
_looks_like_import_path,
binary_is_text,
get_func_args,
get_spec,
memoizemethod_noargs,
to_bytes,
to_unicode,
@ -138,6 +140,11 @@ def test_binaryistext(value: bytes, expected: bool) -> None:
assert binary_is_text(value) is expected
def test_binaryistext_not_bytes() -> None:
with pytest.raises(TypeError, match="data must be bytes, got 'str'"):
binary_is_text("hello") # type: ignore[arg-type]
def test_get_func_args():
def f1(a, b, c):
pass
@ -164,6 +171,8 @@ def test_get_func_args():
partial_f1 = functools.partial(f1, None)
partial_f2 = functools.partial(f1, b=None)
partial_f3 = functools.partial(partial_f2, None)
# a positional value that happens to match the name of a free parameter
partial_f4 = functools.partial(f1, "b")
assert get_func_args(f1) == ["a", "b", "c"]
assert get_func_args(f2) == ["a", "b", "c"]
@ -173,6 +182,7 @@ def test_get_func_args():
assert get_func_args(partial_f1) == ["b", "c"]
assert get_func_args(partial_f2) == ["a", "c"]
assert get_func_args(partial_f3) == ["c"]
assert get_func_args(partial_f4) == ["b", "c"]
assert get_func_args(cal) == ["a", "b", "c"]
assert get_func_args(object) == []
assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"]
@ -209,6 +219,35 @@ def test_get_func_args_unresolvable_annotations():
assert get_func_args(namespace["f"]) == ["a", "b"]
def test_get_func_args_not_callable() -> None:
with pytest.raises(TypeError, match="func must be callable, got 'int'"):
get_func_args(1) # type: ignore[arg-type]
def test_get_spec_not_callable() -> None:
with pytest.raises(TypeError, match="is not callable"):
get_spec(1) # type: ignore[arg-type]
@pytest.mark.parametrize(
("value", "expected"),
[
("scrapy.Spider", True),
("scrapy", True),
("", False),
("scrapy Spider", False),
("scrapy.Spider\n", False),
("scrapy-Spider", False),
(".scrapy", False),
("scrapy.", False),
("scrapy..Spider", False),
("scrapy.1Spider", False),
],
)
def test_looks_like_import_path(value: str, expected: bool) -> None:
assert _looks_like_import_path(value) is expected
@pytest.mark.parametrize(
("value", "expected"),
[

73
tests/test_utils_reactor.py
View File

@ -1,17 +1,29 @@
from __future__ import annotations
import asyncio
import sys
from typing import TYPE_CHECKING
import pytest
from twisted.internet.error import CannotListenError
from twisted.internet.protocol import ServerFactory
from scrapy.utils.reactor import (
_asyncio_reactor_path,
install_reactor,
is_asyncio_reactor_installed,
listen_tcp,
set_asyncio_event_loop,
verify_installed_asyncio_event_loop,
verify_installed_reactor,
)
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import Generator
from twisted.internet.tcp import Port
class TestAsyncio:
@pytest.mark.requires_reactor # needs a reactor
@ -25,3 +37,64 @@ class TestAsyncio:
async def test_set_asyncio_event_loop(self):
install_reactor(_asyncio_reactor_path)
assert set_asyncio_event_loop(None) is asyncio.get_running_loop()
class TestVerifyWithoutReactor:
@pytest.fixture(autouse=True)
def no_reactor(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delitem(sys.modules, "twisted.internet.reactor", raising=False)
def test_verify_installed_reactor(self) -> None:
with pytest.raises(
RuntimeError, match=r"verify_installed_reactor\(\) called without"
):
verify_installed_reactor(_asyncio_reactor_path)
def test_verify_installed_asyncio_event_loop(self) -> None:
with pytest.raises(
RuntimeError,
match=r"verify_installed_asyncio_event_loop\(\) called without",
):
verify_installed_asyncio_event_loop("asyncio.SelectorEventLoop")
@pytest.mark.requires_reactor # needs a reactor
class TestListenTcp:
@pytest.fixture
def ports(self) -> Generator[list[Port]]:
opened: list[Port] = []
yield opened
for port in opened:
port.stopListening()
@pytest.mark.parametrize("portrange", [[1, 2, 3], [8000, 7000]])
def test_invalid_portrange(self, portrange: list[int]) -> None:
with pytest.raises(ValueError, match="invalid portrange"):
listen_tcp(portrange, "127.0.0.1", ServerFactory())
def test_empty_portrange(self, ports: list[Port]) -> None:
port = listen_tcp([], "127.0.0.1", ServerFactory())
ports.append(port)
assert port.getHost().port > 0
def test_single_port(self, ports: list[Port]) -> None:
port = listen_tcp([0], "127.0.0.1", ServerFactory())
ports.append(port)
assert port.getHost().port > 0
def test_skips_used_ports(self, ports: list[Port]) -> None:
used = listen_tcp([], "127.0.0.1", ServerFactory())
ports.append(used)
used_number = used.getHost().port
port = listen_tcp([used_number, used_number + 50], "127.0.0.1", ServerFactory())
ports.append(port)
assert used_number < port.getHost().port <= used_number + 50
def test_no_free_port(self, ports: list[Port]) -> None:
used = listen_tcp([], "127.0.0.1", ServerFactory())
ports.append(used)
used_number = used.getHost().port
with pytest.raises(CannotListenError):
listen_tcp([used_number, used_number], "127.0.0.1", ServerFactory())

26
tests/test_utils_signal.py
View File

@ -53,6 +53,32 @@ class TestSendCatchLog:
dispatcher.disconnect(self.error_handler, signal=test_signal)
dispatcher.disconnect(self.ok_handler, signal=test_signal)
@coroutine_test
async def test_send_catch_log_dont_log(
self, caplog: pytest.LogCaptureFixture
) -> None:
test_signal = object()
handlers_called: set[Callable[..., None]] = set()
dispatcher.connect(self.error_handler, signal=test_signal)
caplog.clear()
result = await ensure_awaitable(
self._get_result(
test_signal,
arg="test",
handlers_called=handlers_called,
dont_log=ZeroDivisionError,
)
)
assert self.error_handler in handlers_called
assert not caplog.records
assert isinstance(
result[0][1], Exception if self.returns_exceptions else Failure
)
dispatcher.disconnect(self.error_handler, signal=test_signal)
def _get_result(self, signal: Any, *a: Any, **kw: Any) -> Any:
return send_catch_log(signal, *a, **kw)

15
tests/test_utils_sitemap.py
View File

@ -41,6 +41,21 @@ def test_sitemap():
]
def test_sitemap_str():
xmltext = """<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.google.com/schemas/sitemap/0.84">
<url><loc>http://www.example.com/</loc></url>
</urlset>"""
with pytest.warns(
ScrapyDeprecationWarning,
match="Passing `str` type as `xmltext` is deprecated",
):
s = Sitemap(xmltext)
assert s.type == "urlset"
assert list(s) == [{"loc": "http://www.example.com/"}]
def test_sitemap_index():
s = Sitemap(
b"""<?xml version="1.0" encoding="UTF-8"?>

58
tests/test_utils_spider.py
View File

@ -1,17 +1,38 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
import pytest
from scrapy import Spider
from scrapy.http import Request
from scrapy.item import Item
from scrapy.utils.spider import iter_spider_classes, iterate_spider_output
from scrapy.settings import Settings
from scrapy.spiderloader import get_spider_loader
from scrapy.utils.spider import (
iter_spider_classes,
iterate_spider_output,
spidercls_for_request,
)
if TYPE_CHECKING:
from scrapy.spiderloader import SpiderLoaderProtocol
class MySpider1(Spider):
name = "myspider1"
allowed_domains = ["example.com", "myspider1.example"]
class MySpider2(Spider):
name = "myspider2"
allowed_domains = ["example.com"]
@pytest.fixture
def spider_loader() -> SpiderLoaderProtocol:
return get_spider_loader(Settings({"SPIDER_MODULES": ["tests.test_utils_spider"]}))
def test_iterate_spider_output():
@ -30,3 +51,38 @@ def test_iter_spider_classes():
it = iter_spider_classes(tests.test_utils_spider)
assert set(it) == {MySpider1, MySpider2}
class TestSpiderclsForRequest:
def test_single_match(self, spider_loader: SpiderLoaderProtocol) -> None:
request = Request("http://myspider1.example/")
assert spidercls_for_request(spider_loader, request) is MySpider1
def test_no_match(self, spider_loader: SpiderLoaderProtocol) -> None:
request = Request("http://toscrape.com/")
assert spidercls_for_request(spider_loader, request) is None
assert spidercls_for_request(spider_loader, request, MySpider1) is MySpider1
def test_multiple_matches(self, spider_loader: SpiderLoaderProtocol) -> None:
request = Request("http://example.com/")
assert spidercls_for_request(spider_loader, request) is None
assert spidercls_for_request(spider_loader, request, MySpider2) is MySpider2
def test_log_none(
self, spider_loader: SpiderLoaderProtocol, caplog: pytest.LogCaptureFixture
) -> None:
request = Request("http://toscrape.com/")
with caplog.at_level(logging.ERROR):
assert spidercls_for_request(spider_loader, request, log_none=True) is None
assert "Unable to find spider that handles" in caplog.text
def test_log_multiple(
self, spider_loader: SpiderLoaderProtocol, caplog: pytest.LogCaptureFixture
) -> None:
request = Request("http://example.com/")
with caplog.at_level(logging.ERROR):
assert (
spidercls_for_request(spider_loader, request, log_multiple=True) is None
)
assert "More than one spider can handle" in caplog.text
assert "myspider1, myspider2" in caplog.text

69
tests/test_utils_ssl.py Normal file
View File

@ -0,0 +1,69 @@
from __future__ import annotations
import logging
import warnings
from typing import cast
from unittest import mock
import OpenSSL._util as pyOpenSSLutil
import OpenSSL.SSL
import pytest
from OpenSSL import crypto
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.ssl import (
_log_ssl_conn_debug_info,
ffi_buf_to_string,
get_openssl_version,
get_temp_key_info,
x509name_to_string,
)
def test_ffi_buf_to_string() -> None:
buf = pyOpenSSLutil.ffi.new("char[]", b"some text")
with pytest.warns(
ScrapyDeprecationWarning, match=r"ffi_buf_to_string\(\) is deprecated"
):
assert ffi_buf_to_string(buf) == "some text"
def test_x509name_to_string() -> None:
with warnings.catch_warnings():
# X509 itself is deprecated in pyOpenSSL, but it is the only way to build
# the X509Name that the deprecated function under test takes.
warnings.simplefilter("ignore", DeprecationWarning)
subject = crypto.X509().get_subject()
subject.C = "IE"
subject.O = "Scrapy"
subject.CN = "localhost"
with pytest.warns(
ScrapyDeprecationWarning, match=r"x509name_to_string\(\) is deprecated"
):
assert x509name_to_string(subject) == "/C=IE/O=Scrapy/CN=localhost"
def test_get_temp_key_info() -> None:
with pytest.warns(
ScrapyDeprecationWarning, match=r"get_temp_key_info\(\) is deprecated"
):
assert get_temp_key_info(object()) is None
def test_get_openssl_version() -> None:
assert "OpenSSL" in get_openssl_version()
def test_log_ssl_conn_debug_info_no_certificate(
caplog: pytest.LogCaptureFixture,
) -> None:
connection = mock.MagicMock()
connection.get_protocol_version_name.return_value = "TLSv1.3"
connection.get_cipher_name.return_value = "TLS_AES_256_GCM_SHA384"
connection.get_peer_certificate.return_value = None
with caplog.at_level(logging.DEBUG, logger="scrapy.utils.ssl"):
_log_ssl_conn_debug_info(
"example.com", cast("OpenSSL.SSL.Connection", connection)
)
assert "SSL connection to example.com using protocol TLSv1.3" in caplog.text
assert "certificate" not in caplog.text

10
tests/test_utils_trackref.py
View File

@ -119,11 +119,21 @@ def test_get_oldest():
assert trackref.get_oldest("Foo") is o3
def test_get_oldest_all_dead():
o1 = Foo()
del o1
if _IS_PYPY:
garbage_collect()
# Foo is still a key of live_refs, but it no longer tracks any instance.
assert trackref.get_oldest("Foo") is None
def test_iter_all():
o1 = Foo()
o2 = Bar() # noqa: F841
o3 = Foo()
assert set(trackref.iter_all("Foo")) == {o1, o3}
assert list(trackref.iter_all("XXX")) == []
def test_run_time_classes() -> None:

30
tests/utils/bases/download_handlers_http.py
View File

@ -31,10 +31,7 @@ from scrapy.exceptions import (
UnsupportedURLSchemeError,
)
from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse
from scrapy.utils._deps_compat import (
PYOPENSSL_X509_DEPRECATED,
TWISTED_TLS_LIMITS_OFFBY1,
)
from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
@ -870,15 +867,8 @@ class TestHttpsBase(TestHttpBase):
is_secure = True
tls_log_message = (
(
'SSL connection certificate: issuer "CN=localhost,O=Scrapy,C=IE", '
'subject "CN=localhost,O=Scrapy,C=IE"'
)
if PYOPENSSL_X509_DEPRECATED
else (
'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", '
'subject "/C=IE/O=Scrapy/CN=localhost"'
)
'SSL connection certificate: issuer "CN=localhost,O=Scrapy,C=IE", '
'subject "CN=localhost,O=Scrapy,C=IE"'
)
def test_download_conn_lost(self) -> None: # type: ignore[override]
@ -940,6 +930,9 @@ class TestSimpleHttpsBase(ABC):
certfile = "keys/localhost.crt"
host = "localhost"
cipher_string: str | None = None
# Crawler settings for the download handler. When unset, the client is asked
# for the same ciphers the server is restricted to.
client_settings: dict[str, Any] | None = None
@pytest.fixture(scope="class")
@classmethod
@ -961,10 +954,9 @@ class TestSimpleHttpsBase(ABC):
@asynccontextmanager
async def get_dh(self) -> AsyncGenerator[DownloadHandlerProtocol]:
if self.cipher_string is not None:
settings_dict = self.client_settings
if settings_dict is None and self.cipher_string is not None:
settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string}
else:
settings_dict = None
crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
crawler.spider = crawler._create_spider()
dh = build_from_crawler(self.download_handler_cls, crawler)
@ -1008,6 +1000,12 @@ class TestHttpsCustomCiphersBase(TestSimpleHttpsBase):
cipher_string = "CAMELLIA256-SHA"
class TestHttpsDefaultCiphersBase(TestSimpleHttpsBase):
"""A ``None`` cipher list leaves the TLS library defaults in place."""
client_settings: dict[str, Any] | None = {"DOWNLOADER_CLIENT_TLS_CIPHERS": None}
class TestHttpsTLSVersionBase(ABC):
keyfile = "keys/localhost.key"
certfile = "keys/localhost.crt"

11
tox.ini
View File

@ -140,15 +140,15 @@ deps =
Twisted==21.7.0
brotli==1.2.0; implementation_name != "pypy"
brotlicffi==1.2.0.0; implementation_name == "pypy"
cryptography==37.0.0
cryptography==41.0.5
cssselect==0.9.1
httpx2==2.0.0
itemadapter==0.1.0
lxml==4.6.4
parsel==1.5.0
pyOpenSSL==22.0.0
pyOpenSSL==24.3.0
queuelib==1.6.1
service_identity==23.1.0
service_identity==24.2.0
w3lib==1.17.0
zope.interface==5.1.0
{[test-requirements]deps}
@ -283,8 +283,7 @@ setenv =
[testenv:pypy3]
basepython = pypy3
commands =
; not enabling coverage as it significantly increases the run time
pytest {posargs:--durations=10 scrapy tests}
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=pypy3.junit.xml -o junit_family=legacy --durations=10 scrapy tests}
[testenv:pypy3-extra-deps]
basepython = pypy3
@ -307,7 +306,7 @@ deps =
parsel==1.5.0
pyOpenSSL==24.3.0
queuelib==1.6.1
service_identity==23.1.0
service_identity==24.2.0
# w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses
# an inline flag placement that Python 3.11 treats as an error: global
# flags not at the start of the expression. w3lib 1.18 stopped encoding []