From f2d6c160070a7696296e19f18b4e813f5332b573 Mon Sep 17 00:00:00 2001 From: Adrian Chaves Date: Fri, 31 Jul 2026 14:22:17 +0200 Subject: [PATCH] Improve test coverage for scrapy.utils --- .github/workflows/tests-ubuntu.yml | 1 + pyproject.toml | 6 +- scrapy/utils/_compression.py | 8 +- scrapy/utils/_deps_compat.py | 2 - scrapy/utils/python.py | 24 +++--- scrapy/utils/ssl.py | 76 ++++--------------- tests/mockserver/utils.py | 10 +-- tests/test_command_shell.py | 23 ++++++ tests/test_crawl.py | 24 +++++- tests/test_downloader_handler_httpx.py | 5 ++ tests/test_downloader_handlers.py | 53 ++++++++++++- ...st_downloadermiddleware_httpcompression.py | 61 ++++++++++++++- tests/test_feedexport_storages.py | 7 ++ tests/test_utils_conf.py | 20 +++++ tests/test_utils_console.py | 9 +++ tests/test_utils_curl.py | 20 +++++ tests/test_utils_defer.py | 9 +++ tests/test_utils_deprecate.py | 19 ++++- tests/test_utils_display.py | 13 +++- tests/test_utils_iterators.py | 34 +++++++++ tests/test_utils_job.py | 24 ++++++ tests/test_utils_log.py | 71 +++++++++++++++++ tests/test_utils_ossignal.py | 54 +++++++++++++ tests/test_utils_project.py | 66 +++++++++++++++- tests/test_utils_python.py | 39 ++++++++++ tests/test_utils_reactor.py | 73 ++++++++++++++++++ tests/test_utils_signal.py | 26 +++++++ tests/test_utils_sitemap.py | 15 ++++ tests/test_utils_spider.py | 58 +++++++++++++- tests/test_utils_ssl.py | 49 ++++++++++++ tests/test_utils_trackref.py | 10 +++ tests/utils/bases/download_handlers_http.py | 30 ++++---- tox.ini | 11 ++- 33 files changed, 829 insertions(+), 121 deletions(-) create mode 100644 tests/test_utils_job.py create mode 100644 tests/test_utils_ossignal.py create mode 100644 tests/test_utils_ssl.py diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index ad2bcfce7..0d8f9fe5c 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -47,6 +47,7 @@ jobs: - python-version: pypy3.11-7.3.20 env: TOXENV: pypy3 + coverage: true # min deps - python-version: "3.10.19" diff --git a/pyproject.toml b/pyproject.toml index 576a42e5c..a1b15eb0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dynamic = ["version"] description = "A high-level Web Crawling and Web Scraping framework" dependencies = [ "Twisted>=21.7.0", - "cryptography>=37.0.0", + "cryptography>=41.0.5", "cssselect>=0.9.1", "defusedxml>=0.7.1", "itemadapter>=0.1.0", @@ -17,9 +17,9 @@ dependencies = [ "packaging", "parsel>=1.5.0", "protego>=0.1.15", - "pyOpenSSL>=22.0.0", + "pyOpenSSL>=24.3.0", "queuelib>=1.4.2", - "service_identity>=23.1.0", + "service_identity>=24.2.0", "tldextract", "w3lib>=1.17.0", "zope.interface>=5.1.0", diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py index 4767c29f2..27716f592 100644 --- a/scrapy/utils/_compression.py +++ b/scrapy/utils/_compression.py @@ -45,17 +45,15 @@ def _inflate(data: bytes, *, max_size: int = 0) -> bytes: _check_max_size(decompressed_size, max_size) output_stream = BytesIO() output_stream.write(first_chunk) - while decompressor.unconsumed_tail: + # Anything left in unconsumed_tail once the stream has ended is not part of + # it, and feeding it back would neither consume it nor produce output. + while decompressor.unconsumed_tail and not decompressor.eof: output_chunk = decompressor.decompress( decompressor.unconsumed_tail, max_length=_CHUNK_SIZE ) decompressed_size += len(output_chunk) _check_max_size(decompressed_size, max_size) output_stream.write(output_chunk) - if tail := decompressor.flush(): - decompressed_size += len(tail) - _check_max_size(decompressed_size, max_size) - output_stream.write(tail) return output_stream.getvalue() diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py index 17957aba4..c6391dd8e 100644 --- a/scrapy/utils/_deps_compat.py +++ b/scrapy/utils/_deps_compat.py @@ -18,8 +18,6 @@ TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0) TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0) PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING) -# pyOpenSSL X.509 APIs are deprecated and cryptography-based ones are preferred -PYOPENSSL_X509_DEPRECATED = PYOPENSSL_VERSION >= Version("24.3.0") # SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0") diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 40fc05257..8485b06f3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -101,7 +101,9 @@ def to_bytes( return text.encode(encoding, errors) -def _chunk_iter(text: str, chunk_size: int) -> Iterable[tuple[str, int]]: +def _chunk_iter( + text: str, chunk_size: int +) -> Iterable[tuple[str, int]]: # pragma: no cover offset = len(text) while True: offset -= chunk_size * 1024 @@ -213,17 +215,15 @@ def get_func_args_dict( except ValueError: return {} - if isinstance(func, partial): - partial_args = func.args - partial_kw = func.keywords - - args = {} - for name, param in sig.parameters.items(): - if name in partial_args: - continue - if partial_kw and name in partial_kw: - continue - args[name] = param + if isinstance(func, partial) and func.keywords: + # The signature of a partial already omits the parameters bound to + # positional arguments, but it keeps those bound to keyword arguments, + # turned into keyword-only parameters with a default. + args = { + name: param + for name, param in sig.parameters.items() + if name not in func.keywords + } else: args = sig.parameters diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 828a99e23..fdb8b8f64 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -11,10 +11,7 @@ import OpenSSL.version from twisted.internet.ssl import CertificateOptions, TLSVersion from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils._deps_compat import ( - PYOPENSSL_X509_DEPRECATED, - TWISTED_TLS_LIMITS_OFFBY1, -) +from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1 from scrapy.utils.python import to_unicode if TYPE_CHECKING: @@ -125,18 +122,17 @@ def _ffi_buf_to_string(buf: Any) -> str: return to_unicode(pyOpenSSLutil.ffi.string(buf)) -def ffi_buf_to_string(buf: Any) -> str: # pragma: no cover +def ffi_buf_to_string(buf: Any) -> str: warnings.warn( "ffi_buf_to_string() is deprecated.", ScrapyDeprecationWarning, stacklevel=2, ) - return ffi_buf_to_string(buf) + return _ffi_buf_to_string(buf) def _x509name_to_string(x509name: X509Name) -> str: # from OpenSSL.crypto.X509Name.__repr__ - # only used on pyOpenSSL < 24.3.0 result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512) pyOpenSSLutil.lib.X509_NAME_oneline( x509name._name, result_buffer, len(result_buffer) @@ -144,7 +140,7 @@ def _x509name_to_string(x509name: X509Name) -> str: return _ffi_buf_to_string(result_buffer) -def x509name_to_string(x509name: X509Name) -> str: # pragma: no cover +def x509name_to_string(x509name: X509Name) -> str: warnings.warn( "x509name_to_string() is deprecated.", ScrapyDeprecationWarning, @@ -153,48 +149,15 @@ def x509name_to_string(x509name: X509Name) -> str: # pragma: no cover return _x509name_to_string(x509name) -def _get_temp_key_info(ssl_object: Any) -> str | None: - # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() - if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"): - # removed in cryptography 40.0.0 (required starting from pyOpenSSL 23.1.0) - return None - temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **") - if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p): - return None - temp_key = temp_key_p[0] - if temp_key == pyOpenSSLutil.ffi.NULL: - return None - temp_key = pyOpenSSLutil.ffi.gc(temp_key, pyOpenSSLutil.lib.EVP_PKEY_free) - key_info = [] - key_type = pyOpenSSLutil.lib.EVP_PKEY_id(temp_key) - if key_type == pyOpenSSLutil.lib.EVP_PKEY_RSA: - key_info.append("RSA") - elif key_type == pyOpenSSLutil.lib.EVP_PKEY_DH: - key_info.append("DH") - elif key_type == pyOpenSSLutil.lib.EVP_PKEY_EC: - key_info.append("ECDH") - ec_key = pyOpenSSLutil.lib.EVP_PKEY_get1_EC_KEY(temp_key) - ec_key = pyOpenSSLutil.ffi.gc(ec_key, pyOpenSSLutil.lib.EC_KEY_free) - nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name( - pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key) - ) - cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid) - if cname == pyOpenSSLutil.ffi.NULL: - cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid) - key_info.append(_ffi_buf_to_string(cname)) - else: - key_info.append(_ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) - key_info.append(f"{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits") - return ", ".join(key_info) - - -def get_temp_key_info(ssl_object: Any) -> str | None: # pragma: no cover +def get_temp_key_info(ssl_object: Any) -> str | None: + # A no-op: it read the negotiated ephemeral key through a binding that + # cryptography 40.0.0 removed. warnings.warn( "get_temp_key_info() is deprecated. It's also a no-op with cryptography 40.0.0+.", ScrapyDeprecationWarning, stacklevel=2, ) - return _get_temp_key_info(ssl_object) + return None def get_openssl_version() -> str: @@ -210,23 +173,12 @@ def _log_ssl_conn_debug_info(hostname: str, connection: OpenSSL.SSL.Connection) connection.get_protocol_version_name(), connection.get_cipher_name(), ) - if PYOPENSSL_X509_DEPRECATED: - if server_cert := connection.get_peer_certificate(as_cryptography=True): - logger.debug( - 'SSL connection certificate: issuer "%s", subject "%s"', - server_cert.issuer.rfc4514_string(), - server_cert.subject.rfc4514_string(), - ) - else: # noqa: PLR5501 - if server_cert_pyopenssl := connection.get_peer_certificate(): - logger.debug( - 'SSL connection certificate: issuer "%s", subject "%s"', - _x509name_to_string(server_cert_pyopenssl.get_issuer()), - _x509name_to_string(server_cert_pyopenssl.get_subject()), - ) - key_info = _get_temp_key_info(connection._ssl) - if key_info: - logger.debug("SSL temp key: %s", key_info) + if server_cert := connection.get_peer_certificate(as_cryptography=True): + logger.debug( + 'SSL connection certificate: issuer "%s", subject "%s"', + server_cert.issuer.rfc4514_string(), + server_cert.subject.rfc4514_string(), + ) # Twisted-specific diff --git a/tests/mockserver/utils.py b/tests/mockserver/utils.py index 33c3340b9..e993bedb2 100644 --- a/tests/mockserver/utils.py +++ b/tests/mockserver/utils.py @@ -7,11 +7,9 @@ from typing import TYPE_CHECKING, cast from cryptography.hazmat.primitives.serialization import load_pem_private_key from cryptography.x509 import load_pem_x509_certificate from OpenSSL import SSL -from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey from twisted.internet.ssl import CertificateOptions, ContextFactory from scrapy.core.downloader.tls import _TWISTED_VERSION_MAP -from scrapy.utils._deps_compat import PYOPENSSL_X509_DEPRECATED from scrapy.utils.python import to_bytes from scrapy.utils.ssl import _get_cert_options_version_kwargs @@ -37,12 +35,8 @@ def ssl_context_factory( keyfile_path = Path(__file__).parent.parent / keyfile certfile_path = Path(__file__).parent.parent / certfile - if PYOPENSSL_X509_DEPRECATED: - cert = load_pem_x509_certificate(certfile_path.read_bytes()) - key = load_pem_private_key(keyfile_path.read_bytes(), password=None) - else: - cert = load_certificate(FILETYPE_PEM, certfile_path.read_bytes()) # type: ignore[assignment] - key = load_privatekey(FILETYPE_PEM, keyfile_path.read_bytes()) # type: ignore[assignment] + cert = load_pem_x509_certificate(certfile_path.read_bytes()) + key = load_pem_private_key(keyfile_path.read_bytes(), password=None) tls_min = _TWISTED_VERSION_MAP.get(tls_min_version) if tls_min_version else None tls_max = _TWISTED_VERSION_MAP.get(tls_max_version) if tls_max_version else None diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 29667a1ae..91032a892 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -276,6 +276,29 @@ class TestInteractiveShell: self._isolate_config(env, config_home) assert "Traceback" not in self._run_interactive_shell(env) + @pytest.mark.skipif( + importlib.util.find_spec("IPython") is None, reason="IPython is not installed" + ) + def test_shell_ipython(self, tmp_path: Path) -> None: + # Reaching the embedded IPython shell requires selecting it explicitly, + # since ptpython takes precedence when both are installed. + config_home = tmp_path / "config" + config_home.mkdir() + env = os.environ.copy() + self._isolate_config(env, config_home) + env["SCRAPY_PYTHON_SHELL"] = "ipython" + args = (sys.executable, "-m", "scrapy.cmdline", "shell") + logfile = BytesIO() + p = PopenSpawn(args, env=env, timeout=30) + p.logfile_read = logfile + p.expect_exact("Available Scrapy objects") + p.sendline("import sys; print('IPYMODULE', 'IPython' in sys.modules)") + p.expect_exact("IPYMODULE True") + p.sendeof() + p.wait() # type: ignore[no-untyped-call] + logfile.seek(0) + assert "Traceback" not in logfile.read().decode() + @pytest.fixture def restore_sigint(): diff --git a/tests/test_crawl.py b/tests/test_crawl.py index d284805be..862714167 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -2,6 +2,7 @@ from __future__ import annotations import json import logging +import re from ipaddress import IPv4Address from socket import gethostbyname from typing import TYPE_CHECKING, Any @@ -18,7 +19,11 @@ from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownloa from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.defer import maybe_deferred_to_future -from scrapy.utils.engine import format_engine_status, get_engine_status +from scrapy.utils.engine import ( + format_engine_status, + get_engine_status, + print_engine_status, +) from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE @@ -372,6 +377,23 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == 1 + @coroutine_test + async def test_print_engine_status( + self, mockserver: MockServer, capsys: pytest.CaptureFixture[str] + ) -> None: + def cb(response): + assert crawler.engine + print_engine_status(crawler.engine) + + crawler = get_crawler(SingleRequestSpider) + await crawler.crawl_async( + seed=mockserver.url("/"), callback_func=cb, mockserver=mockserver + ) + assert isinstance(crawler.spider, SingleRequestSpider) + out = capsys.readouterr().out + assert out.startswith("Execution engine status") + assert re.search(rf"engine\.spider\.name +: {crawler.spider.name}\n", out) + @coroutine_test async def test_format_engine_status(self, mockserver: MockServer) -> None: est: list[str] = [] diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index 5ceb93382..07b48a5c7 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -20,6 +20,7 @@ from tests.utils.bases.download_handlers_http import ( TestHttpProxyBase, TestHttpsBase, TestHttpsCustomCiphersBase, + TestHttpsDefaultCiphersBase, TestHttpsInvalidDNSIdBase, TestHttpsInvalidDNSPatternBase, TestHttpsTLSVersionBase, @@ -133,6 +134,10 @@ class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBa pass +class TestHttpsDefaultCiphers(HttpxDownloadHandlerMixin, TestHttpsDefaultCiphersBase): + pass + + class TestHttpsTLSVersion(HttpxDownloadHandlerMixin, TestHttpsTLSVersionBase): pass diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 8cb9308ff..8f5c13d62 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -9,15 +9,31 @@ from tempfile import mkdtemp, mkstemp from unittest import mock import pytest +from twisted.internet.defer import CancelledError +from twisted.internet.error import ConnectionRefusedError as TxConnectionRefusedError +from twisted.internet.error import DNSLookupError +from twisted.internet.error import TimeoutError as TxTimeoutError +from twisted.web.client import ResponseFailed +from twisted.web.error import SchemeNotSupported from w3lib.url import path_to_file_uri from scrapy.core.downloader.handlers import DownloadHandlers from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler from scrapy.core.downloader.handlers.file import FileDownloadHandler from scrapy.core.downloader.handlers.s3 import S3DownloadHandler -from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning +from scrapy.exceptions import ( + CannotResolveHostError, + DownloadCancelledError, + DownloadConnectionRefusedError, + DownloadFailedError, + DownloadTimeoutError, + NotConfigured, + ScrapyDeprecationWarning, + UnsupportedURLSchemeError, +) from scrapy.http import Request, TextResponse from scrapy.responsetypes import responsetypes +from scrapy.utils._download_handlers import wrap_twisted_exceptions from scrapy.utils.boto import is_botocore_available from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler @@ -407,3 +423,38 @@ class TestDataURI: request = Request("data:,") response = await self.download_request(request) assert response.protocol is None + + +class TestWrapTwistedExceptions: + """Every Twisted exception the HTTP handlers can raise has a Scrapy equivalent. + + Most mappings are also covered end to end by the HTTP handler tests, but a + Twisted ``TimeoutError`` needs a connection attempt that is silently dropped + rather than refused, which no mock server can offer reliably. + """ + + @pytest.mark.parametrize( + ("twisted_exception", "scrapy_exception"), + [ + (SchemeNotSupported, UnsupportedURLSchemeError), + (CancelledError, DownloadCancelledError), + (TxConnectionRefusedError, DownloadConnectionRefusedError), + (DNSLookupError, CannotResolveHostError), + (ResponseFailed, DownloadFailedError), + (TxTimeoutError, DownloadTimeoutError), + ], + ) + def test_mapping( + self, + twisted_exception: type[Exception], + scrapy_exception: type[Exception], + ) -> None: + original = twisted_exception("some message") + with pytest.raises(scrapy_exception) as exc_info, wrap_twisted_exceptions(): + raise original + assert exc_info.value.__cause__ is original + assert str(exc_info.value) == str(original) + + def test_other_exceptions_pass_through(self) -> None: + with pytest.raises(ZeroDivisionError), wrap_twisted_exceptions(): + 1 / 0 diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 55f06b396..70ee05d76 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,8 +1,10 @@ +import zlib from gzip import GzipFile from importlib.util import find_spec from io import BytesIO from logging import WARNING from pathlib import Path +from threading import Thread import pytest from w3lib.encoding import resolve_encoding @@ -15,6 +17,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWar from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes from scrapy.spiders import Spider +from scrapy.utils._compression import _CHUNK_SIZE from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler from tests import tests_datadir @@ -513,6 +516,57 @@ class TestHttpCompression: self.assertStatsEqual("httpcompression/response_count", None) self.assertStatsEqual("httpcompression/response_bytes", None) + @pytest.mark.parametrize( + "extra", + [ + pytest.param(b"garbage", id="trailing-garbage"), + pytest.param(zlib.compress(b"b" * 100_000), id="second-zlib-stream"), + ], + ) + def test_deflate_extra_data_after_stream(self, extra: bytes) -> None: + """Bytes after the end of a deflate stream must not stall decompression. + + Such bytes stay in ``unconsumed_tail`` and produce no output, so feeding + them back to the decompressor never makes progress. Decompressing in a + thread lets the test fail instead of hanging if that ever regresses. + """ + # The body must decompress to more than _CHUNK_SIZE, or the whole stream + # is consumed by the first decompress() call and no loop is entered. + plain = b"a" * 100_000 + response = Response( + "http://example.com", + body=zlib.compress(plain) + extra, + headers={"Content-Encoding": "deflate"}, + ) + result: list[Response] = [] + thread = Thread( + target=lambda: result.append( + self.mw.process_response(Request("http://example.com"), response) + ), + daemon=True, + ) + thread.start() + thread.join(60) + assert not thread.is_alive(), "decompression did not terminate" + assert result[0].body == plain + + def test_deflate_chunked_output_leaves_nothing_to_flush(self) -> None: + """Guard the zlib behaviour that lets ``_inflate()`` skip ``flush()``. + + ``decompress(max_length=...)`` leaves the input it could not turn into + output in ``unconsumed_tail``, so once that is empty every complete + stream has been fully emitted. If a future zlib buffers output instead, + this fails and ``_inflate()`` needs to flush the remainder again. + """ + for size in (_CHUNK_SIZE - 1, _CHUNK_SIZE, _CHUNK_SIZE + 1, 4 * _CHUNK_SIZE): + decompressor = zlib.decompressobj() + decompressor.decompress(zlib.compress(b"a" * size), max_length=_CHUNK_SIZE) + while decompressor.unconsumed_tail and not decompressor.eof: + decompressor.decompress( + decompressor.unconsumed_tail, max_length=_CHUNK_SIZE + ) + assert decompressor.flush() == b"", f"pending output for size {size}" + def _test_compression_bomb_setting(self, compression_id): settings = {"DOWNLOAD_MAXSIZE": 1_000_000} crawler = get_crawler(Spider, settings_dict=settings) @@ -523,7 +577,12 @@ class TestHttpCompression: response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B with pytest.raises(IgnoreRequest) as exc_info: mw.process_response(response.request, response) - assert exc_info.value.__cause__.decompressed_size < 1_100_000 + cause = exc_info.value.__cause__ + assert cause.decompressed_size < 1_100_000 + assert str(cause) == ( + f"The number of bytes decompressed so far ({cause.decompressed_size} B) " + "exceeded the specified maximum (1000000 B)." + ) def test_compression_bomb_setting_br(self): _skip_if_no_br() diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index 4d28872b7..63511206b 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -162,6 +162,13 @@ class TestFTPFeedStorage: await self._store(url, b"bar", settings=settings) self._assert_stored(ftp_server.path / filename, b"bar") + @coroutine_test + async def test_missing_parent_directories(self): + with MockFTPServer() as ftp_server: + path = "missing/parent/dirs/file" + await self._store(ftp_server.url(path), b"foo") + self._assert_stored(ftp_server.path / path, b"foo") + def test_uri_auth_quote(self): # RFC3986: 3.2.1. User Information pw_quoted = quote(string.punctuation, safe="") diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py index 518cc3518..89f5cd141 100644 --- a/tests/test_utils_conf.py +++ b/tests/test_utils_conf.py @@ -9,8 +9,10 @@ from scrapy.settings import BaseSettings, Settings from scrapy.utils.conf import ( arglist_to_dict, build_component_list, + closest_scrapy_cfg, feed_complete_default_values_from_settings, feed_process_params_from_cli, + get_sources, ) @@ -39,6 +41,20 @@ class TestBuildComponentList: ): build_component_list(duplicate_bs, convert=lambda x: x.lower()) + def test_duplicate_components_in_dict(self): + d = {"one": 1, "ONE": 2} + with pytest.raises( + ValueError, match=r"Some paths in .* convert to the same object" + ): + build_component_list(d, convert=lambda x: x.lower()) + + def test_invalid_value(self): + d = {"one": "1"} + with pytest.raises( + ValueError, match=r"Invalid value 1 for component one, please provide" + ): + build_component_list(d, convert=lambda x: x) + def test_valid_numbers(self): # work well with None and numeric values d = {"a": 10, "b": None, "c": 15, "d": 5.0} @@ -51,6 +67,10 @@ class TestBuildComponentList: assert build_component_list(d, convert=lambda x: x) == ["b", "c", "a"] +def test_get_sources(): + assert get_sources() == [*get_sources(use_closest=False), closest_scrapy_cfg()] + + def test_arglist_to_dict(): assert arglist_to_dict(["arg1=val1", "arg2=val2"]) == { "arg1": "val1", diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py index ab0c72d8a..73a382e7d 100644 --- a/tests/test_utils_console.py +++ b/tests/test_utils_console.py @@ -16,6 +16,15 @@ def test_get_shell_embed_func(): assert shell.__name__ == "_embed_standard_shell" +def test_get_shell_embed_func_known_shells(): + def embed(namespace: dict[str, object] | None = None, banner: str = "") -> None: + pass + + known_shells = {"custom": lambda: embed} + assert get_shell_embed_func(["custom"], known_shells) is embed + assert get_shell_embed_func(["python"], known_shells) is None + + def test_get_shell_embed_func_python(): # the standard shell is always available shell = get_shell_embed_func(["python"]) diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py index 6b30744bb..8ab6c5c2f 100644 --- a/tests/test_utils_curl.py +++ b/tests/test_utils_curl.py @@ -41,6 +41,26 @@ class TestCurlToRequestKwargs: } self._test_command(curl_command, expected_result) + def test_get_cookie_option(self): + curl_command = 'curl "http://example.org/" -b "a=1; b=2"' + expected_result = { + "method": "GET", + "url": "http://example.org/", + "cookies": {"a": "1", "b": "2"}, + } + self._test_command(curl_command, expected_result) + + def test_get_cookie_file_option(self): + # curl reads cookies from a file when the value is not a key-value + # pair; Scrapy ignores it. + curl_command = 'curl "http://example.org/" -b cookies.txt -b "a=1"' + expected_result = { + "method": "GET", + "url": "http://example.org/", + "cookies": {"a": "1"}, + } + self._test_command(curl_command, expected_result) + def test_get_complex(self): curl_command = ( "curl 'http://httpbin.org/get' -H 'Accept-Encoding: gzip, deflate'" diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 175a4fe03..3327b4052 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -357,6 +357,15 @@ class TestDeferredToFuture: assert future_result == 42 +@pytest.mark.only_not_asyncio +class TestDeferredToFutureNotAsyncio: + def test_deferred(self): + with pytest.raises( + RuntimeError, match=r"deferred_to_future\(\) requires an installed asyncio" + ): + deferred_to_future(Deferred()) + + @pytest.mark.only_asyncio class TestMaybeDeferredToFutureAsyncio: @coroutine_test diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 4c8585916..c877f5a9e 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -7,7 +7,7 @@ from unittest import mock import pytest from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.deprecate import create_deprecated_class, update_classpath +from scrapy.utils.deprecate import attribute, create_deprecated_class, update_classpath class MyWarning(UserWarning): @@ -284,3 +284,20 @@ class TestUpdateClassPath: def test_returns_nonstring(self): for notastring in [None, True, [1, 2, 3], object()]: assert update_classpath(notastring) == notastring + + +class TestAttribute: + class MyClass: + pass + + def test_default_version(self): + with pytest.warns( + ScrapyDeprecationWarning, + match=r"MyClass\.old attribute is deprecated and will be no longer " + r"supported in Scrapy 0\.12, use MyClass\.new attribute instead", + ): + attribute(self.MyClass(), "old", "new") + + def test_custom_version(self): + with pytest.warns(ScrapyDeprecationWarning, match="in Scrapy 3.0"): + attribute(self.MyClass(), "old", "new", "3.0") diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py index a87816c20..b8d15bcba 100644 --- a/tests/test_utils_display.py +++ b/tests/test_utils_display.py @@ -1,11 +1,14 @@ from __future__ import annotations import builtins +import sys from io import StringIO from typing import TYPE_CHECKING from unittest import mock -from scrapy.utils.display import pformat, pprint +import pytest + +from scrapy.utils.display import _enable_windows_terminal_processing, pformat, pprint if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -81,6 +84,14 @@ def test_pformat_windows( assert pformat(value) in colorized_strings +@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only console API") +def test_enable_windows_terminal_processing() -> None: + # The tests above mock this out, so this is the only place the real + # kernel32 calls run. Whether they succeed depends on whether stdout is a + # console, which pytest does not guarantee. + assert isinstance(_enable_windows_terminal_processing(), bool) + + @mock.patch("sys.platform", "linux") @mock.patch("sys.stdout.isatty") def test_pformat_no_pygments(isatty: mock.Mock) -> None: diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py index d431b8017..672ae0be6 100644 --- a/tests/test_utils_iterators.py +++ b/tests/test_utils_iterators.py @@ -210,6 +210,31 @@ class TestXmliter: with pytest.raises(StopIteration): next(my_iter) + def test_xmliter_namespaced_nodename_other_prefixes(self): + body = b""" + + + A_1 + ITEM_1 + + + """ + response = XmlResponse(url="http://example.com", body=body) + node = next(xmliter_lxml(response, "g:id")) + assert node.xpath("text()").getall() == ["ITEM_1"] + + def test_xmliter_non_text_response(self): + body = ( + b'' + b"one" + ) + response = Response(url="http://example.com", body=body) + assert [ + node.xpath("text()").get() for node in xmliter_lxml(response, "product") + ] == ["one"] + def test_xmliter_exception(self): body = ( '' @@ -421,6 +446,9 @@ class TestUtilsCsv: {"id": "4", "name": "empty", "value": ""}, ] + def test_csviter_empty(self): + assert not list(csviter("")) + def test_csviter_exception(self): body = get_testdata("feeds", "feed-sample3.csv") @@ -481,6 +509,12 @@ class TestBodyOrStr: assert type(r1) is type(r2) assert type(r1) is not type(r3) # type: ignore[comparison-overlap] + def test_body_or_str_unsupported_type(self) -> None: + with pytest.raises( + TypeError, match="must be Response or str or bytes, not int" + ): + _body_or_str(42) # type: ignore[call-overload] + @staticmethod def _assert_type_and_value( a: str | bytes, b: str | bytes, obj: Response | str | bytes diff --git a/tests/test_utils_job.py b/tests/test_utils_job.py new file mode 100644 index 000000000..31adc1514 --- /dev/null +++ b/tests/test_utils_job.py @@ -0,0 +1,24 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from scrapy.settings import Settings +from scrapy.utils.job import job_dir + +if TYPE_CHECKING: + from pathlib import Path + + +def test_no_jobdir() -> None: + assert job_dir(Settings()) is None + assert job_dir(Settings({"JOBDIR": ""})) is None + + +def test_existing_jobdir(tmp_path: Path) -> None: + assert job_dir(Settings({"JOBDIR": str(tmp_path)})) == str(tmp_path) + + +def test_missing_jobdir(tmp_path: Path) -> None: + jobdir = tmp_path / "missing" / "jobdir" + assert job_dir(Settings({"JOBDIR": str(jobdir)})) == str(jobdir) + assert jobdir.is_dir() diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index 7f5301387..085779c61 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -4,18 +4,25 @@ import json import logging import re import sys +import warnings from io import StringIO from typing import TYPE_CHECKING, Any import pytest +from twisted.python import log as twisted_log from twisted.python.failure import Failure +from scrapy.settings import Settings from scrapy.utils.log import ( LogCounterHandler, SpiderLoggerAdapter, StreamLogger, TopLevelFormatter, + _uninstall_scrapy_root_handler, + configure_logging, failure_to_exc_info, + get_scrapy_root_handler, + install_scrapy_root_handler, ) from scrapy.utils.test import get_crawler from tests.spiders import LogSpider @@ -118,6 +125,70 @@ class TestStreamLogger: sys.stdout = old_stdout + def test_flush(self) -> None: + class FlushCountingHandler(logging.Handler): + flushes = 0 + + def flush(self) -> None: + self.flushes += 1 + + handler = FlushCountingHandler() + logger = logging.getLogger("test_flush") + logger.addHandler(handler) + try: + StreamLogger(logger).flush() + finally: + logger.removeHandler(handler) + assert handler.flushes == 1 + + +class TestConfigureLogging: + @pytest.fixture(autouse=True) + def restore_logging(self) -> Generator[None]: + root_handlers = logging.root.handlers[:] + # configure_logging() adds a Twisted log observer without giving any way + # to remove it afterwards. + observers = twisted_log.theLogPublisher.observers[:] + old_stdout = sys.stdout + old_showwarning = warnings.showwarning + try: + yield + finally: + warnings.showwarning = old_showwarning + sys.stdout = old_stdout + twisted_log.theLogPublisher.observers[:] = observers + logging.root.handlers[:] = root_handlers + _uninstall_scrapy_root_handler() + + @staticmethod + def _warnings_are_captured() -> bool: + return warnings.showwarning.__module__ == "logging" + + def test_log_stdout(self) -> None: + configure_logging(settings={"LOG_STDOUT": True}, install_root_handler=False) + assert isinstance(sys.stdout, StreamLogger) + + def test_captures_warnings(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "warnoptions", []) + logging.captureWarnings(False) + configure_logging(install_root_handler=False) + assert self._warnings_are_captured() + + def test_keeps_warnoptions(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys, "warnoptions", ["default"]) + logging.captureWarnings(False) + configure_logging(install_root_handler=False) + assert not self._warnings_are_captured() + + def test_reinstall_root_handler_removed_from_root(self) -> None: + install_scrapy_root_handler(Settings()) + handler = get_scrapy_root_handler() + assert handler is not None + # Something else removed the handler from the root logger. + logging.root.removeHandler(handler) + install_scrapy_root_handler(Settings()) + assert get_scrapy_root_handler() is not handler + @pytest.mark.parametrize( ("base_extra", "log_extra", "expected_extra"), diff --git a/tests/test_utils_ossignal.py b/tests/test_utils_ossignal.py new file mode 100644 index 000000000..4c8f7d0b4 --- /dev/null +++ b/tests/test_utils_ossignal.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import signal +from typing import TYPE_CHECKING, Any + +import pytest + +from scrapy.utils.ossignal import install_shutdown_handlers + +if TYPE_CHECKING: + from collections.abc import Generator + from types import FrameType + +SHUTDOWN_SIGNALS = [ + getattr(signal, name) + for name in ("SIGTERM", "SIGINT", "SIGBREAK") + if hasattr(signal, name) +] + + +@pytest.fixture(autouse=True) +def restore_handlers() -> Generator[None]: + handlers = {sig: signal.getsignal(sig) for sig in SHUTDOWN_SIGNALS} + try: + yield + finally: + for sig, handler in handlers.items(): + signal.signal(sig, handler) + + +def shutdown(signum: int, frame: FrameType | None) -> Any: + pass + + +def test_install() -> None: + install_shutdown_handlers(shutdown) + for sig in SHUTDOWN_SIGNALS: + assert signal.getsignal(sig) is shutdown + + +def test_keeps_custom_sigint_handler() -> None: + def custom(signum: int, frame: FrameType | None) -> Any: + pass + + signal.signal(signal.SIGINT, custom) + install_shutdown_handlers(shutdown, override_sigint=False) + assert signal.getsignal(signal.SIGINT) is custom + assert signal.getsignal(signal.SIGTERM) is shutdown + + +def test_overrides_default_sigint_handler() -> None: + signal.signal(signal.SIGINT, signal.default_int_handler) + install_shutdown_handlers(shutdown, override_sigint=False) + assert signal.getsignal(signal.SIGINT) is shutdown diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py index 7eade6463..8a7f2ee03 100644 --- a/tests/test_utils_project.py +++ b/tests/test_utils_project.py @@ -6,8 +6,14 @@ from typing import TYPE_CHECKING import pytest +from scrapy.exceptions import NotConfigured from scrapy.utils.misc import set_environ -from scrapy.utils.project import data_path, get_project_settings +from scrapy.utils.project import ( + data_path, + get_project_settings, + inside_project, + project_data_dir, +) if TYPE_CHECKING: from collections.abc import Generator @@ -27,6 +33,19 @@ def proj_path(tmp_path: Path) -> Generator[Path]: os.chdir(prev_dir) +@pytest.fixture +def no_proj_path(tmp_path: Path) -> Generator[Path]: + """A working directory without a scrapy.cfg file, also isolated from the + user-wide and system-wide Scrapy configuration files.""" + prev_dir = Path.cwd() + try: + os.chdir(tmp_path) + with set_environ(HOME=str(tmp_path), XDG_CONFIG_HOME=str(tmp_path)): + yield tmp_path + finally: + os.chdir(prev_dir) + + def test_data_path_outside_project() -> None: assert str(Path(".scrapy", "somepath")) == data_path("somepath") abspath = str(Path(os.path.sep, "absolute", "path")) @@ -40,6 +59,51 @@ def test_data_path_inside_project(proj_path: Path) -> None: assert abspath == data_path(abspath) +def test_data_path_createdir(no_proj_path: Path) -> None: + path = Path(data_path("somepath", createdir=True)) + assert path.is_dir() + # An existing directory is left alone. + assert Path(data_path("somepath", createdir=True)) == path + + +def test_inside_project_unimportable_settings_module(no_proj_path: Path) -> None: + with ( + set_environ(SCRAPY_SETTINGS_MODULE="tests.no_such_settings_module"), + pytest.warns( + UserWarning, match="Cannot import scrapy settings module tests.no_such" + ), + ): + assert inside_project() is False + + +def test_project_data_dir_outside_project(no_proj_path: Path) -> None: + with pytest.raises(NotConfigured, match="Not inside a project"): + project_data_dir() + + +def test_project_data_dir_without_scrapy_cfg(no_proj_path: Path) -> None: + with ( + set_environ(SCRAPY_SETTINGS_MODULE="tests.test_cmdline.settings"), + pytest.raises(NotConfigured, match=r"Unable to find scrapy\.cfg file"), + ): + project_data_dir() + + +def test_project_data_dir_default(proj_path: Path) -> None: + expected = (proj_path / ".scrapy").resolve() + assert Path(project_data_dir()) == expected + assert expected.is_dir() + # A second call finds the directory already created. + assert Path(project_data_dir()) == expected + + +def test_project_data_dir_from_scrapy_cfg(proj_path: Path) -> None: + datadir = proj_path / "custom-datadir" + Path("scrapy.cfg").write_text(f"[datadir]\ndefault = {datadir}\n") + assert Path(project_data_dir()) == datadir + assert datadir.is_dir() + + class TestGetProjectSettings: def test_valid_envvar(self): value = "tests.test_cmdline.settings" diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 099b5ccc2..25e50960f 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -12,8 +12,10 @@ from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import aiter_errback from scrapy.utils.python import ( MutableAsyncChain, + _looks_like_import_path, binary_is_text, get_func_args, + get_spec, memoizemethod_noargs, to_bytes, to_unicode, @@ -138,6 +140,11 @@ def test_binaryistext(value: bytes, expected: bool) -> None: assert binary_is_text(value) is expected +def test_binaryistext_not_bytes() -> None: + with pytest.raises(TypeError, match="data must be bytes, got 'str'"): + binary_is_text("hello") # type: ignore[arg-type] + + def test_get_func_args(): def f1(a, b, c): pass @@ -164,6 +171,8 @@ def test_get_func_args(): partial_f1 = functools.partial(f1, None) partial_f2 = functools.partial(f1, b=None) partial_f3 = functools.partial(partial_f2, None) + # a positional value that happens to match the name of a free parameter + partial_f4 = functools.partial(f1, "b") assert get_func_args(f1) == ["a", "b", "c"] assert get_func_args(f2) == ["a", "b", "c"] @@ -173,6 +182,7 @@ def test_get_func_args(): assert get_func_args(partial_f1) == ["b", "c"] assert get_func_args(partial_f2) == ["a", "c"] assert get_func_args(partial_f3) == ["c"] + assert get_func_args(partial_f4) == ["b", "c"] assert get_func_args(cal) == ["a", "b", "c"] assert get_func_args(object) == [] assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"] @@ -209,6 +219,35 @@ def test_get_func_args_unresolvable_annotations(): assert get_func_args(namespace["f"]) == ["a", "b"] +def test_get_func_args_not_callable() -> None: + with pytest.raises(TypeError, match="func must be callable, got 'int'"): + get_func_args(1) # type: ignore[arg-type] + + +def test_get_spec_not_callable() -> None: + with pytest.raises(TypeError, match="is not callable"): + get_spec(1) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + ("scrapy.Spider", True), + ("scrapy", True), + ("", False), + ("scrapy Spider", False), + ("scrapy.Spider\n", False), + ("scrapy-Spider", False), + (".scrapy", False), + ("scrapy.", False), + ("scrapy..Spider", False), + ("scrapy.1Spider", False), + ], +) +def test_looks_like_import_path(value: str, expected: bool) -> None: + assert _looks_like_import_path(value) is expected + + @pytest.mark.parametrize( ("value", "expected"), [ diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py index 44cb5c306..3be0a03b8 100644 --- a/tests/test_utils_reactor.py +++ b/tests/test_utils_reactor.py @@ -1,17 +1,29 @@ from __future__ import annotations import asyncio +import sys +from typing import TYPE_CHECKING import pytest +from twisted.internet.error import CannotListenError +from twisted.internet.protocol import ServerFactory from scrapy.utils.reactor import ( _asyncio_reactor_path, install_reactor, is_asyncio_reactor_installed, + listen_tcp, set_asyncio_event_loop, + verify_installed_asyncio_event_loop, + verify_installed_reactor, ) from tests.utils.decorators import coroutine_test +if TYPE_CHECKING: + from collections.abc import Generator + + from twisted.internet.tcp import Port + class TestAsyncio: @pytest.mark.requires_reactor # needs a reactor @@ -25,3 +37,64 @@ class TestAsyncio: async def test_set_asyncio_event_loop(self): install_reactor(_asyncio_reactor_path) assert set_asyncio_event_loop(None) is asyncio.get_running_loop() + + +class TestVerifyWithoutReactor: + @pytest.fixture(autouse=True) + def no_reactor(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delitem(sys.modules, "twisted.internet.reactor", raising=False) + + def test_verify_installed_reactor(self) -> None: + with pytest.raises( + RuntimeError, match=r"verify_installed_reactor\(\) called without" + ): + verify_installed_reactor(_asyncio_reactor_path) + + def test_verify_installed_asyncio_event_loop(self) -> None: + with pytest.raises( + RuntimeError, + match=r"verify_installed_asyncio_event_loop\(\) called without", + ): + verify_installed_asyncio_event_loop("asyncio.SelectorEventLoop") + + +@pytest.mark.requires_reactor # needs a reactor +class TestListenTcp: + @pytest.fixture + def ports(self) -> Generator[list[Port]]: + opened: list[Port] = [] + yield opened + for port in opened: + port.stopListening() + + @pytest.mark.parametrize("portrange", [[1, 2, 3], [8000, 7000]]) + def test_invalid_portrange(self, portrange: list[int]) -> None: + with pytest.raises(ValueError, match="invalid portrange"): + listen_tcp(portrange, "127.0.0.1", ServerFactory()) + + def test_empty_portrange(self, ports: list[Port]) -> None: + port = listen_tcp([], "127.0.0.1", ServerFactory()) + ports.append(port) + assert port.getHost().port > 0 + + def test_single_port(self, ports: list[Port]) -> None: + port = listen_tcp([0], "127.0.0.1", ServerFactory()) + ports.append(port) + assert port.getHost().port > 0 + + def test_skips_used_ports(self, ports: list[Port]) -> None: + used = listen_tcp([], "127.0.0.1", ServerFactory()) + ports.append(used) + used_number = used.getHost().port + + port = listen_tcp([used_number, used_number + 50], "127.0.0.1", ServerFactory()) + ports.append(port) + assert used_number < port.getHost().port <= used_number + 50 + + def test_no_free_port(self, ports: list[Port]) -> None: + used = listen_tcp([], "127.0.0.1", ServerFactory()) + ports.append(used) + used_number = used.getHost().port + + with pytest.raises(CannotListenError): + listen_tcp([used_number, used_number], "127.0.0.1", ServerFactory()) diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 3109bc149..cbcc070d5 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -53,6 +53,32 @@ class TestSendCatchLog: dispatcher.disconnect(self.error_handler, signal=test_signal) dispatcher.disconnect(self.ok_handler, signal=test_signal) + @coroutine_test + async def test_send_catch_log_dont_log( + self, caplog: pytest.LogCaptureFixture + ) -> None: + test_signal = object() + handlers_called: set[Callable[..., None]] = set() + + dispatcher.connect(self.error_handler, signal=test_signal) + caplog.clear() + result = await ensure_awaitable( + self._get_result( + test_signal, + arg="test", + handlers_called=handlers_called, + dont_log=ZeroDivisionError, + ) + ) + + assert self.error_handler in handlers_called + assert not caplog.records + assert isinstance( + result[0][1], Exception if self.returns_exceptions else Failure + ) + + dispatcher.disconnect(self.error_handler, signal=test_signal) + def _get_result(self, signal: Any, *a: Any, **kw: Any) -> Any: return send_catch_log(signal, *a, **kw) diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py index 9f2dce6d6..dda13006d 100644 --- a/tests/test_utils_sitemap.py +++ b/tests/test_utils_sitemap.py @@ -41,6 +41,21 @@ def test_sitemap(): ] +def test_sitemap_str(): + xmltext = """ + +http://www.example.com/ +""" + + with pytest.warns( + ScrapyDeprecationWarning, + match="Passing `str` type as `xmltext` is deprecated", + ): + s = Sitemap(xmltext) + assert s.type == "urlset" + assert list(s) == [{"loc": "http://www.example.com/"}] + + def test_sitemap_index(): s = Sitemap( b""" diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py index 71f0f76ad..6caa1f6a3 100644 --- a/tests/test_utils_spider.py +++ b/tests/test_utils_spider.py @@ -1,17 +1,38 @@ from __future__ import annotations +import logging +from typing import TYPE_CHECKING + +import pytest + from scrapy import Spider from scrapy.http import Request from scrapy.item import Item -from scrapy.utils.spider import iter_spider_classes, iterate_spider_output +from scrapy.settings import Settings +from scrapy.spiderloader import get_spider_loader +from scrapy.utils.spider import ( + iter_spider_classes, + iterate_spider_output, + spidercls_for_request, +) + +if TYPE_CHECKING: + from scrapy.spiderloader import SpiderLoaderProtocol class MySpider1(Spider): name = "myspider1" + allowed_domains = ["example.com", "myspider1.example"] class MySpider2(Spider): name = "myspider2" + allowed_domains = ["example.com"] + + +@pytest.fixture +def spider_loader() -> SpiderLoaderProtocol: + return get_spider_loader(Settings({"SPIDER_MODULES": ["tests.test_utils_spider"]})) def test_iterate_spider_output(): @@ -30,3 +51,38 @@ def test_iter_spider_classes(): it = iter_spider_classes(tests.test_utils_spider) assert set(it) == {MySpider1, MySpider2} + + +class TestSpiderclsForRequest: + def test_single_match(self, spider_loader: SpiderLoaderProtocol) -> None: + request = Request("http://myspider1.example/") + assert spidercls_for_request(spider_loader, request) is MySpider1 + + def test_no_match(self, spider_loader: SpiderLoaderProtocol) -> None: + request = Request("http://toscrape.com/") + assert spidercls_for_request(spider_loader, request) is None + assert spidercls_for_request(spider_loader, request, MySpider1) is MySpider1 + + def test_multiple_matches(self, spider_loader: SpiderLoaderProtocol) -> None: + request = Request("http://example.com/") + assert spidercls_for_request(spider_loader, request) is None + assert spidercls_for_request(spider_loader, request, MySpider2) is MySpider2 + + def test_log_none( + self, spider_loader: SpiderLoaderProtocol, caplog: pytest.LogCaptureFixture + ) -> None: + request = Request("http://toscrape.com/") + with caplog.at_level(logging.ERROR): + assert spidercls_for_request(spider_loader, request, log_none=True) is None + assert "Unable to find spider that handles" in caplog.text + + def test_log_multiple( + self, spider_loader: SpiderLoaderProtocol, caplog: pytest.LogCaptureFixture + ) -> None: + request = Request("http://example.com/") + with caplog.at_level(logging.ERROR): + assert ( + spidercls_for_request(spider_loader, request, log_multiple=True) is None + ) + assert "More than one spider can handle" in caplog.text + assert "myspider1, myspider2" in caplog.text diff --git a/tests/test_utils_ssl.py b/tests/test_utils_ssl.py new file mode 100644 index 000000000..146293711 --- /dev/null +++ b/tests/test_utils_ssl.py @@ -0,0 +1,49 @@ +from __future__ import annotations + +import warnings + +import OpenSSL._util as pyOpenSSLutil +import pytest +from OpenSSL import crypto + +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.ssl import ( + ffi_buf_to_string, + get_openssl_version, + get_temp_key_info, + x509name_to_string, +) + + +def test_ffi_buf_to_string() -> None: + buf = pyOpenSSLutil.ffi.new("char[]", b"some text") + with pytest.warns( + ScrapyDeprecationWarning, match=r"ffi_buf_to_string\(\) is deprecated" + ): + assert ffi_buf_to_string(buf) == "some text" + + +def test_x509name_to_string() -> None: + with warnings.catch_warnings(): + # X509 itself is deprecated in pyOpenSSL, but it is the only way to build + # the X509Name that the deprecated function under test takes. + warnings.simplefilter("ignore", DeprecationWarning) + subject = crypto.X509().get_subject() + subject.C = "IE" + subject.O = "Scrapy" + subject.CN = "localhost" + with pytest.warns( + ScrapyDeprecationWarning, match=r"x509name_to_string\(\) is deprecated" + ): + assert x509name_to_string(subject) == "/C=IE/O=Scrapy/CN=localhost" + + +def test_get_temp_key_info() -> None: + with pytest.warns( + ScrapyDeprecationWarning, match=r"get_temp_key_info\(\) is deprecated" + ): + assert get_temp_key_info(object()) is None + + +def test_get_openssl_version() -> None: + assert "OpenSSL" in get_openssl_version() diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py index 5458aa603..bb8f95c3e 100644 --- a/tests/test_utils_trackref.py +++ b/tests/test_utils_trackref.py @@ -119,8 +119,18 @@ def test_get_oldest(): assert trackref.get_oldest("Foo") is o3 +def test_get_oldest_all_dead(): + o1 = Foo() + del o1 + if _IS_PYPY: + garbage_collect() + # Foo is still a key of live_refs, but it no longer tracks any instance. + assert trackref.get_oldest("Foo") is None + + def test_iter_all(): o1 = Foo() o2 = Bar() # noqa: F841 o3 = Foo() assert set(trackref.iter_all("Foo")) == {o1, o3} + assert list(trackref.iter_all("XXX")) == [] diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py index e44f9bcb8..4a3e75e13 100644 --- a/tests/utils/bases/download_handlers_http.py +++ b/tests/utils/bases/download_handlers_http.py @@ -31,10 +31,7 @@ from scrapy.exceptions import ( UnsupportedURLSchemeError, ) from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse -from scrapy.utils._deps_compat import ( - PYOPENSSL_X509_DEPRECATED, - TWISTED_TLS_LIMITS_OFFBY1, -) +from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1 from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.misc import build_from_crawler from scrapy.utils.spider import DefaultSpider @@ -832,15 +829,8 @@ class TestHttpsBase(TestHttpBase): is_secure = True tls_log_message = ( - ( - 'SSL connection certificate: issuer "CN=localhost,O=Scrapy,C=IE", ' - 'subject "CN=localhost,O=Scrapy,C=IE"' - ) - if PYOPENSSL_X509_DEPRECATED - else ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' - 'subject "/C=IE/O=Scrapy/CN=localhost"' - ) + 'SSL connection certificate: issuer "CN=localhost,O=Scrapy,C=IE", ' + 'subject "CN=localhost,O=Scrapy,C=IE"' ) def test_download_conn_lost(self) -> None: # type: ignore[override] @@ -902,6 +892,9 @@ class TestSimpleHttpsBase(ABC): certfile = "keys/localhost.crt" host = "localhost" cipher_string: str | None = None + # Crawler settings for the download handler. When unset, the client is asked + # for the same ciphers the server is restricted to. + client_settings: dict[str, Any] | None = None @pytest.fixture(scope="class") @classmethod @@ -923,10 +916,9 @@ class TestSimpleHttpsBase(ABC): @asynccontextmanager async def get_dh(self) -> AsyncGenerator[DownloadHandlerProtocol]: - if self.cipher_string is not None: + settings_dict = self.client_settings + if settings_dict is None and self.cipher_string is not None: settings_dict = {"DOWNLOADER_CLIENT_TLS_CIPHERS": self.cipher_string} - else: - settings_dict = None crawler = get_crawler(DefaultSpider, settings_dict=settings_dict) crawler.spider = crawler._create_spider() dh = build_from_crawler(self.download_handler_cls, crawler) @@ -970,6 +962,12 @@ class TestHttpsCustomCiphersBase(TestSimpleHttpsBase): cipher_string = "CAMELLIA256-SHA" +class TestHttpsDefaultCiphersBase(TestSimpleHttpsBase): + """A ``None`` cipher list leaves the TLS library defaults in place.""" + + client_settings: dict[str, Any] | None = {"DOWNLOADER_CLIENT_TLS_CIPHERS": None} + + class TestHttpsTLSVersionBase(ABC): keyfile = "keys/localhost.key" certfile = "keys/localhost.crt" diff --git a/tox.ini b/tox.ini index 18e1579c9..b922fd9d7 100644 --- a/tox.ini +++ b/tox.ini @@ -134,15 +134,15 @@ deps = pytest==8.4.0 Protego==0.1.15 Twisted==21.7.0 - cryptography==37.0.0 + cryptography==41.0.5 cssselect==0.9.1 httpx2==2.0.0 itemadapter==0.1.0 lxml==4.6.4 parsel==1.5.0 - pyOpenSSL==22.0.0 + pyOpenSSL==24.3.0 queuelib==1.4.2 - service_identity==23.1.0 + service_identity==24.2.0 w3lib==1.17.0 zope.interface==5.1.0 {[test-requirements]deps} @@ -236,8 +236,7 @@ setenv = [testenv:pypy3] basepython = pypy3 commands = - ; not enabling coverage as it significantly increases the run time - pytest {posargs:--durations=10 scrapy tests} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=pypy3.junit.xml -o junit_family=legacy --durations=10 scrapy tests} [testenv:pypy3-extra-deps] basepython = pypy3 @@ -260,7 +259,7 @@ deps = parsel==1.5.0 pyOpenSSL==24.3.0 queuelib==1.4.2 - service_identity==23.1.0 + service_identity==24.2.0 # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses # an inline flag placement that Python 3.11 treats as an error: global # flags not at the start of the expression. w3lib 1.18 stopped encoding []