diff --git a/conftest.py b/conftest.py
index 31d818f22..603c06f7f 100644
--- a/conftest.py
+++ b/conftest.py
@@ -74,40 +74,19 @@ def mockserver() -> Generator[MockServer]:
@pytest.fixture # function scope because it modifies os.environ
-def mitm_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
- proxy = MitmProxy()
+def proxy_server(
+ request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch
+) -> Generator[str]:
+ kind = request.param
+ proxy = MitmProxy(mode="socks5" if kind == "socks5" else None)
url = proxy.start()
+ if kind == "https":
+ url = url.replace("http://", "https://")
monkeypatch.setenv("http_proxy", url)
monkeypatch.setenv("https_proxy", url)
try:
- yield proxy
- finally:
- proxy.stop()
-
-
-@pytest.fixture # function scope because it modifies os.environ
-def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
- proxy = MitmProxy()
- url = proxy.start().replace("http://", "https://")
- monkeypatch.setenv("http_proxy", url)
- monkeypatch.setenv("https_proxy", url)
-
- try:
- yield proxy
- finally:
- proxy.stop()
-
-
-@pytest.fixture # function scope because it modifies os.environ
-def socks5_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
- proxy = MitmProxy(mode="socks5")
- url = proxy.start()
- monkeypatch.setenv("http_proxy", url)
- monkeypatch.setenv("https_proxy", url)
-
- try:
- yield proxy
+ yield kind
finally:
proxy.stop()
diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py
index 16b655716..5a669c732 100644
--- a/scrapy/core/downloader/handlers/_base_streaming.py
+++ b/scrapy/core/downloader/handlers/_base_streaming.py
@@ -241,6 +241,16 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon
body=response_body.getvalue(),
)
+ @staticmethod
+ def _request_headers(request: Request) -> Headers:
+ """Get a prepared copy of the request headers.
+
+ This removes the Proxy-Authorization header.
+ """
+ headers = request.headers.copy()
+ headers.pop(b"Proxy-Authorization", None)
+ return headers
+
def _get_bind_address_host(self) -> str | None:
"""Return the host portion of the bind address.
@@ -279,10 +289,8 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon
if not proxy:
return None, None
proxy = add_http_if_no_scheme(proxy)
- auth_header: list[bytes] | None = request.headers.pop(
- b"Proxy-Authorization", None
- )
- return proxy, auth_header[0].decode("ascii") if auth_header else None
+ auth_header: bytes | None = request.headers.get(b"Proxy-Authorization")
+ return proxy, auth_header.decode("ascii") if auth_header else None
def _extract_proxy_url_with_creds(self, request: Request) -> str | None:
"""Return the proxy URL with the userinfo added based on the
diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py
index c960fc152..9b54b44d8 100644
--- a/scrapy/core/downloader/handlers/_httpx.py
+++ b/scrapy/core/downloader/handlers/_httpx.py
@@ -152,13 +152,14 @@ class HttpxDownloadHandler(_Base):
f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed."
)
client = self._get_client(proxy)
+ headers = self._request_headers(request).to_tuple_list()
try:
async with client.stream(
request.method,
request.url,
content=request.body,
- headers=request.headers.to_tuple_list(),
+ headers=headers,
timeout=timeout,
) as response:
yield response
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 83d04e6ca..762b0fdf1 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -11,14 +11,20 @@ import hashlib
import warnings
from contextlib import suppress
from io import BytesIO
-from typing import TYPE_CHECKING, Any, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar, cast
from itemadapter import ItemAdapter
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
-from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum
+from scrapy.pipelines.files import (
+ FileException,
+ FilesPipeline,
+ GCSFilesStore,
+ S3FilesStore,
+ _md5sum,
+)
from scrapy.utils.defer import ensure_awaitable
from scrapy.utils.python import to_bytes
@@ -33,6 +39,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
+ from scrapy.settings import BaseSettings
class ImageException(FileException):
@@ -126,6 +133,20 @@ class ImagesPipeline(FilesPipeline):
) -> str:
return await self.image_downloaded(response, request, info, item=item)
+ @classmethod
+ def _update_stores(cls, settings: BaseSettings) -> None:
+ super()._update_stores(settings)
+
+ s3store: type[S3FilesStore] = cast(
+ "type[S3FilesStore]", cls.STORE_SCHEMES["s3"]
+ )
+ s3store.POLICY = settings["IMAGES_STORE_S3_ACL"]
+
+ gcs_store: type[GCSFilesStore] = cast(
+ "type[GCSFilesStore]", cls.STORE_SCHEMES["gs"]
+ )
+ gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None
+
async def image_downloaded(
self,
response: Response,
diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py
index fad7e6f6b..17957aba4 100644
--- a/scrapy/utils/_deps_compat.py
+++ b/scrapy/utils/_deps_compat.py
@@ -1,7 +1,15 @@
+import sys
+
from OpenSSL import __version__ as PYOPENSSL_VERSION_STRING
from packaging.version import Version
from twisted import version as TWISTED_VERSION
from twisted.python.versions import Version as TxVersion
+from w3lib import __version__ as W3LIB_VERSION_STRING
+
+# improved urllib.robotparser, https://github.com/python/cpython/pull/149374
+STDLIB_IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5) or (
+ (3, 13, 14) <= sys.version_info < (3, 14)
+)
TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0)
# changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506
@@ -10,7 +18,11 @@ TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0)
PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)
-# SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object
-PYOPENSSL_WANTS_X509_PKEY = PYOPENSSL_VERSION < Version("24.3.0")
+# pyOpenSSL X.509 APIs are deprecated and cryptography-based ones are preferred
+PYOPENSSL_X509_DEPRECATED = PYOPENSSL_VERSION >= Version("24.3.0")
# SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable
PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0")
+
+W3LIB_VERSION = Version(W3LIB_VERSION_STRING)
+# safe_url_string() strips the input, https://github.com/scrapy/w3lib/pull/207
+W3LIB_STRIPS_URLS = W3LIB_VERSION >= Version("2.1.1")
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 0b67eaa34..47568e656 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -162,7 +162,7 @@ def md5sum(file: IO[bytes]) -> str:
def rel_has_nofollow(rel: str | None) -> bool:
"""Return True if link rel attribute has nofollow type"""
- return rel is not None and "nofollow" in rel.replace(",", " ").split()
+ return rel is not None and "nofollow" in rel.lower().replace(",", " ").split()
class SupportsFromCrawler(Protocol[_T_co, _P]):
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index ffb7fae49..398403d90 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -179,6 +179,12 @@ def _get_method(obj: Any, name: Any) -> Any:
raise ValueError(f"Method {name!r} not found in: {obj}") from None
+def _cookie_value_to_unicode(value: str | bytes | float) -> str:
+ if isinstance(value, bytes):
+ return value.decode()
+ return str(value)
+
+
def request_to_curl(request: Request) -> str:
"""
Converts a :class:`~scrapy.Request` object to a curl command.
@@ -202,7 +208,9 @@ def request_to_curl(request: Request) -> str:
cookies = f"--cookie '{cookie}'"
elif isinstance(request.cookies, list):
cookie = "; ".join(
- f"{next(iter(c.keys()))}={next(iter(c.values()))}"
+ f"{_cookie_value_to_unicode(c['name'])}={_cookie_value_to_unicode(c['value'])}"
+ if "name" in c and "value" in c
+ else f"{next(iter(c.keys()))}={next(iter(c.values()))}"
for c in request.cookies
)
cookies = f"--cookie '{cookie}'"
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index 22e9414b8..828a99e23 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import logging
import ssl
+import warnings
from typing import TYPE_CHECKING, Any, TypedDict, TypeVar
import OpenSSL._util as pyOpenSSLutil
@@ -9,7 +10,11 @@ import OpenSSL.SSL
import OpenSSL.version
from twisted.internet.ssl import CertificateOptions, TLSVersion
-from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils._deps_compat import (
+ PYOPENSSL_X509_DEPRECATED,
+ TWISTED_TLS_LIMITS_OFFBY1,
+)
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
@@ -116,23 +121,42 @@ def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None:
# pyOpenSSL utils
-def ffi_buf_to_string(buf: Any) -> str:
+def _ffi_buf_to_string(buf: Any) -> str:
return to_unicode(pyOpenSSLutil.ffi.string(buf))
-def x509name_to_string(x509name: X509Name) -> str:
+def ffi_buf_to_string(buf: Any) -> str: # pragma: no cover
+ warnings.warn(
+ "ffi_buf_to_string() is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return ffi_buf_to_string(buf)
+
+
+def _x509name_to_string(x509name: X509Name) -> str:
# from OpenSSL.crypto.X509Name.__repr__
+ # only used on pyOpenSSL < 24.3.0
result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512)
pyOpenSSLutil.lib.X509_NAME_oneline(
x509name._name, result_buffer, len(result_buffer)
)
- return ffi_buf_to_string(result_buffer)
+ return _ffi_buf_to_string(result_buffer)
-def get_temp_key_info(ssl_object: Any) -> str | None:
+def x509name_to_string(x509name: X509Name) -> str: # pragma: no cover
+ warnings.warn(
+ "x509name_to_string() is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return _x509name_to_string(x509name)
+
+
+def _get_temp_key_info(ssl_object: Any) -> str | None:
# adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()
if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"):
- # removed in cryptography 40.0.0
+ # removed in cryptography 40.0.0 (required starting from pyOpenSSL 23.1.0)
return None
temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")
if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p):
@@ -157,13 +181,22 @@ def get_temp_key_info(ssl_object: Any) -> str | None:
cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid)
if cname == pyOpenSSLutil.ffi.NULL:
cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid)
- key_info.append(ffi_buf_to_string(cname))
+ key_info.append(_ffi_buf_to_string(cname))
else:
- key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
+ key_info.append(_ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
key_info.append(f"{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits")
return ", ".join(key_info)
+def get_temp_key_info(ssl_object: Any) -> str | None: # pragma: no cover
+ warnings.warn(
+ "get_temp_key_info() is deprecated. It's also a no-op with cryptography 40.0.0+.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return _get_temp_key_info(ssl_object)
+
+
def get_openssl_version() -> str:
system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)
system_openssl = system_openssl_bytes.decode("ascii", errors="replace")
@@ -177,14 +210,21 @@ def _log_ssl_conn_debug_info(hostname: str, connection: OpenSSL.SSL.Connection)
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
- server_cert = connection.get_peer_certificate()
- if server_cert:
- logger.debug(
- 'SSL connection certificate: issuer "%s", subject "%s"',
- x509name_to_string(server_cert.get_issuer()),
- x509name_to_string(server_cert.get_subject()),
- )
- key_info = get_temp_key_info(connection._ssl)
+ if PYOPENSSL_X509_DEPRECATED:
+ if server_cert := connection.get_peer_certificate(as_cryptography=True):
+ logger.debug(
+ 'SSL connection certificate: issuer "%s", subject "%s"',
+ server_cert.issuer.rfc4514_string(),
+ server_cert.subject.rfc4514_string(),
+ )
+ else: # noqa: PLR5501
+ if server_cert_pyopenssl := connection.get_peer_certificate():
+ logger.debug(
+ 'SSL connection certificate: issuer "%s", subject "%s"',
+ _x509name_to_string(server_cert_pyopenssl.get_issuer()),
+ _x509name_to_string(server_cert_pyopenssl.get_subject()),
+ )
+ key_info = _get_temp_key_info(connection._ssl)
if key_info:
logger.debug("SSL temp key: %s", key_info)
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 8f75e2618..4d2bbdda2 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -139,7 +139,8 @@ def strip_url(
("ftp", 21),
}
):
- netloc = netloc.replace(f":{parsed_url.port}", "")
+ port_suffix = f":{parsed_url.port}"
+ netloc = netloc.removesuffix(port_suffix)
return urlunparse(
(
diff --git a/tests/ignores.txt b/tests/ignores.txt
index 222288841..94edcf186 100644
--- a/tests/ignores.txt
+++ b/tests/ignores.txt
@@ -1,3 +1 @@
-scrapy/downloadermiddlewares/cookies.py
scrapy/extensions/statsmailer.py
-scrapy/extensions/memusage.py
diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.py
index 943775fa5..fdea666e1 100644
--- a/tests/mockserver/simple_https.py
+++ b/tests/mockserver/simple_https.py
@@ -33,7 +33,7 @@ class SimpleMockServer(BaseMockServer):
super().__init__()
self.keyfile = keyfile
self.certfile = certfile
- self.cipher_string = cipher_string or ""
+ self.cipher_string = cipher_string
self.tls_min_version = tls_min_version
self.tls_max_version = tls_max_version
diff --git a/tests/mockserver/utils.py b/tests/mockserver/utils.py
index 7aa656780..5c4ca7457 100644
--- a/tests/mockserver/utils.py
+++ b/tests/mockserver/utils.py
@@ -10,7 +10,7 @@ from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey
from twisted.internet.ssl import CertificateOptions, ContextFactory
from scrapy.core.downloader.tls import _TWISTED_VERSION_MAP
-from scrapy.utils._deps_compat import PYOPENSSL_WANTS_X509_PKEY
+from scrapy.utils._deps_compat import PYOPENSSL_X509_DEPRECATED
from scrapy.utils.python import to_bytes
from scrapy.utils.ssl import _get_cert_options_version_kwargs
@@ -29,7 +29,7 @@ def ssl_context_factory(
keyfile_path = Path(__file__).parent.parent / keyfile
certfile_path = Path(__file__).parent.parent / certfile
- if not PYOPENSSL_WANTS_X509_PKEY:
+ if PYOPENSSL_X509_DEPRECATED:
cert = load_pem_x509_certificate(certfile_path.read_bytes())
key = load_pem_private_key(keyfile_path.read_bytes(), password=None)
else:
diff --git a/tests/spiders.py b/tests/spiders.py
index 55d1ea365..612dc11c9 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -222,7 +222,7 @@ class AsyncDefDeferredWrappedSpider(SimpleSpider):
class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
- name = "asyncdef_deferred_wrapped"
+ name = "asyncdef_deferred_maybe_wrapped"
async def parse(self, response):
await maybe_deferred_to_future(defer.succeed(None))
diff --git a/tests/test_addons.py b/tests/test_addons.py
index db0fb2f31..14ebddda8 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -161,6 +161,7 @@ class TestAddonManager:
settings.set("KEY", 0, priority="default")
runner = runner_cls(settings)
crawler = runner.create_crawler(Spider)
+ crawler._apply_settings()
assert crawler.settings.getint("KEY") == 20
def test_fallback_workflow(self):
diff --git a/tests/test_cmdline/extensions.py b/tests/test_cmdline/extensions.py
index 11c821f8d..ef1e50c0c 100644
--- a/tests/test_cmdline/extensions.py
+++ b/tests/test_cmdline/extensions.py
@@ -1,14 +1,2 @@
-"""A test extension used to check the settings loading order"""
-
-
-class TestExtension:
- def __init__(self, settings):
- settings.set("TEST1", f"{settings['TEST1']} + started")
-
- @classmethod
- def from_crawler(cls, crawler):
- return cls(crawler.settings)
-
-
class DummyExtension:
pass
diff --git a/tests/test_cmdline/settings.py b/tests/test_cmdline/settings.py
index 32b15e191..ec71bba0c 100644
--- a/tests/test_cmdline/settings.py
+++ b/tests/test_cmdline/settings.py
@@ -1,7 +1,7 @@
from pathlib import Path
EXTENSIONS = {
- "tests.test_cmdline.extensions.TestExtension": 0,
+ "tests.test_cmdline.extensions.DummyExtension": 0,
}
TEST1 = "default"
diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py
index 2a9d0ed57..7ac6c4fb0 100644
--- a/tests/test_command_startproject.py
+++ b/tests/test_command_startproject.py
@@ -257,7 +257,7 @@ class TestStartprojectTemplates:
assert actual_permissions == expected_permissions
- def test_startproject_permissions_umask_022(self, tmp_path: Path) -> None:
+ def test_startproject_permissions_umask_002(self, tmp_path: Path) -> None:
"""Check that generated files have the right permissions when the
system uses a umask value that causes new files to have different
permissions than those from the template folder."""
diff --git a/tests/test_commands.py b/tests/test_commands.py
index edb03da1b..0657f393c 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -214,9 +214,7 @@ class MySpider(scrapy.Spider):
self._append_settings(proj_path / self.project_name, "TWISTED_REACTOR = None\n")
self._assert_spider_works(self.NORMAL_MSG, proj_path, "sp")
- self._assert_spider_asyncio_fail(
- self.NORMAL_MSG, proj_path, "aiosp", "-s", "TWISTED_REACTOR="
- )
+ self._assert_spider_asyncio_fail(self.NORMAL_MSG, proj_path, "aiosp")
def test_spider_settings_asyncio(self, proj_path: Path) -> None:
"""The reactor is set via the spider settings to the asyncio value.
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 008e326ec..e80945b93 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -388,7 +388,7 @@ class TestContractsManager:
request = self.conman.from_method(spider.returns_item_meta, self.results)
assert request.meta["key"] == "example"
response.meta = request.meta
- request.callback(ResponseMetaMock)
+ request.callback(response)
assert response.meta["key"] == "example"
self.should_succeed()
@@ -476,14 +476,14 @@ class TestContractsManager:
# invalid regex
request = self.conman.from_method(spider.invalid_regex, self.results)
- self.should_succeed()
+ assert request is None
# invalid regex with valid contract
request = self.conman.from_method(
spider.invalid_regex_with_valid_contract, self.results
)
- self.should_succeed()
request.callback(response)
+ self.should_succeed()
def test_custom_contracts(self):
self.conman.from_spider(CustomContractSuccessSpider(), self.results)
@@ -578,7 +578,7 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = CustomFailContractPreProcess(spider.returns_request)
- conman = ContractsManager([contract])
+ conman = ContractsManager([UrlContract, ReturnsContract, contract])
request = conman.from_method(spider.returns_request, self.results)
contract.add_pre_hook(request, self.results)
@@ -592,7 +592,7 @@ class TestCustomContractPrePostProcess:
spider = DemoSpider()
response = ResponseMock()
contract = CustomFailContractPostProcess(spider.returns_request)
- conman = ContractsManager([contract])
+ conman = ContractsManager([UrlContract, ReturnsContract, contract])
request = conman.from_method(spider.returns_request, self.results)
contract.add_post_hook(request, self.results)
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index abeaa2f65..e348bfb7f 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -36,7 +36,6 @@ from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
- from twisted.internet.ssl import ContextFactory
from twisted.web.iweb import IBodyProducer
@@ -48,8 +47,6 @@ class TestSlot:
@pytest.mark.requires_reactor # this test is related to the Twisted HTTP code
class TestContextFactoryBase:
- context_factory: ContextFactory | None = None
-
@async_yield_fixture
async def server_url(self, tmp_path):
(tmp_path / "file").write_bytes(b"0123456789")
@@ -69,7 +66,7 @@ class TestContextFactoryBase:
return reactor.listenSSL(
0,
site,
- contextFactory=self.context_factory or ssl_context_factory(),
+ contextFactory=ssl_context_factory(),
interface="127.0.0.1",
)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index ada4c31ce..85d98f847 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -342,7 +342,7 @@ with multiples lines
assert "responses" in crawler.spider.meta
assert "failures" not in crawler.spider.meta
# start() doesn't set Referer header
- echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][2].body))
+ echo0 = json.loads(to_unicode(crawler.spider.meta["responses"][0].body))
assert "Referer" not in echo0["headers"]
# following request sets Referer to the source request url
echo1 = json.loads(to_unicode(crawler.spider.meta["responses"][1].body))
@@ -390,7 +390,7 @@ with multiples lines
est = [x for sublist in est for x in sublist] # flatten
est = [x.lstrip().rstrip() for x in est]
it = iter(est)
- s = dict(zip(it, it, strict=False))
+ s = dict(zip(it, it, strict=True))
assert s["engine.spider.name"] == crawler.spider.name
assert s["len(engine.scraper.slot.active)"] == "1"
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 82956735b..31d195c4b 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -240,11 +240,7 @@ class TestCrawler(TestBaseCrawler):
@classmethod
def from_crawler(cls, crawler):
- try:
- crawler.get_downloader_middleware(DefaultSpider)
- except Exception as e:
- MySpider.result = e
- raise
+ crawler.get_downloader_middleware(DefaultSpider)
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with pytest.raises(RuntimeError):
@@ -322,11 +318,7 @@ class TestCrawler(TestBaseCrawler):
@classmethod
def from_crawler(cls, crawler):
- try:
- crawler.get_extension(DefaultSpider)
- except Exception as e:
- MySpider.result = e
- raise
+ crawler.get_extension(DefaultSpider)
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with pytest.raises(RuntimeError):
@@ -404,11 +396,7 @@ class TestCrawler(TestBaseCrawler):
@classmethod
def from_crawler(cls, crawler):
- try:
- crawler.get_item_pipeline(DefaultSpider)
- except Exception as e:
- MySpider.result = e
- raise
+ crawler.get_item_pipeline(DefaultSpider)
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with pytest.raises(RuntimeError):
@@ -486,11 +474,7 @@ class TestCrawler(TestBaseCrawler):
@classmethod
def from_crawler(cls, crawler):
- try:
- crawler.get_spider_middleware(DefaultSpider)
- except Exception as e:
- MySpider.result = e
- raise
+ crawler.get_spider_middleware(DefaultSpider)
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with pytest.raises(RuntimeError):
@@ -755,11 +739,12 @@ class TestCrawlerRunnerHasSpider:
):
await self._crawl(runner, NoRequestsSpider)
else:
- CrawlerRunner(
+ runner = CrawlerRunner(
settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
)
+ await self._crawl(runner, NoRequestsSpider)
@pytest.mark.only_asyncio
diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py
index 361e91382..489b70e74 100644
--- a/tests/test_downloader_handler_twisted_ftp.py
+++ b/tests/test_downloader_handler_twisted_ftp.py
@@ -59,7 +59,7 @@ class TestFTPBase(ABC):
port = reactor.listenTCP(0, factory, interface="127.0.0.1")
portno = port.getHost().port
- yield f"https://127.0.0.1:{portno}/"
+ yield f"ftp://127.0.0.1:{portno}/"
await port.stopListening()
@@ -142,15 +142,11 @@ class TestFTPBase(ABC):
server_url: str,
dh: FTPDownloadHandler,
) -> None:
- f, local_fname = mkstemp()
- local_fname_path = Path(local_fname)
- os.close(f)
meta = {}
meta.update(self.req_meta)
request = Request(url=server_url + filename, meta=meta)
r = await dh.download_request(request)
assert type(r) is response_class # pylint: disable=unidiomatic-typecheck
- local_fname_path.unlink()
class TestFTP(TestFTPBase):
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 5f79a5453..bea97642e 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -24,6 +24,7 @@ from tests.test_downloader_handlers_http_base import (
TestHttpWithCrawlerBase,
TestMitmProxyBase,
TestRealWebsiteBase,
+ TestSimpleHttpsBase,
)
from tests.utils.decorators import coroutine_test
@@ -156,6 +157,10 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
await download_handler.download_request(request)
+class TestSimpleHttp2(H2DownloadHandlerMixin, TestSimpleHttpsBase):
+ pass
+
+
class TestHttp2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase):
pass
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index eadb7740e..e685f607a 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -191,17 +191,14 @@ class TestS3:
@contextlib.contextmanager
def _mocked_date(self, date):
- try:
- import botocore.auth # noqa: F401,PLC0415
- except ImportError:
+ import botocore.auth # noqa: F401,PLC0415
+
+ # We need to mock botocore.auth.formatdate, because otherwise
+ # botocore overrides Date header with current date and time
+ # and Authorization header is different each time
+ with mock.patch("botocore.auth.formatdate") as mock_formatdate:
+ mock_formatdate.return_value = date
yield
- else:
- # We need to mock botocore.auth.formatdate, because otherwise
- # botocore overrides Date header with current date and time
- # and Authorization header is different each time
- with mock.patch("botocore.auth.formatdate") as mock_formatdate:
- mock_formatdate.return_value = date
- yield
@coroutine_test
async def test_request_signing1(self):
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index 0e1ff07c9..2ae9206a0 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -33,7 +33,10 @@ from scrapy.exceptions import (
UnsupportedURLSchemeError,
)
from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse
-from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1
+from scrapy.utils._deps_compat import (
+ PYOPENSSL_X509_DEPRECATED,
+ TWISTED_TLS_LIMITS_OFFBY1,
+)
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
@@ -563,7 +566,7 @@ class TestHttpBase(ABC):
) -> None:
request = Request(mockserver.url("/text", is_secure=self.is_secure))
- # 10 is minimal size for this request and the limit is only counted on
+ # 5 is minimal size for this request and the limit is only counted on
# response body. (regardless of headers)
async with self.get_dh({"DOWNLOAD_MAXSIZE": 5}) as download_handler:
response = await download_handler.download_request(request)
@@ -831,8 +834,15 @@ class TestHttpsBase(TestHttpBase):
is_secure = True
tls_log_message = (
- 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", '
- 'subject "/C=IE/O=Scrapy/CN=localhost"'
+ (
+ 'SSL connection certificate: issuer "CN=localhost,O=Scrapy,C=IE", '
+ 'subject "CN=localhost,O=Scrapy,C=IE"'
+ )
+ if PYOPENSSL_X509_DEPRECATED
+ else (
+ 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", '
+ 'subject "/C=IE/O=Scrapy/CN=localhost"'
+ )
)
def test_download_conn_lost(self) -> None: # type: ignore[override]
@@ -896,16 +906,17 @@ class TestSimpleHttpsBase(ABC):
cipher_string: str | None = None
@pytest.fixture(scope="class")
- def simple_mockserver(self) -> Generator[SimpleMockServer]:
+ @classmethod
+ def simple_mockserver(cls) -> Generator[SimpleMockServer]:
with SimpleMockServer(
- self.keyfile, self.certfile, cipher_string=self.cipher_string
+ cls.keyfile, cls.certfile, cipher_string=cls.cipher_string
) as simple_mockserver:
yield simple_mockserver
@pytest.fixture(scope="class")
- def url(self, simple_mockserver: SimpleMockServer) -> str:
- # need to use self.host instead of what mockserver returns
- return f"https://{self.host}:{simple_mockserver.port(is_secure=True)}/file"
+ @classmethod
+ def url(cls, simple_mockserver: SimpleMockServer) -> str:
+ return f"https://{cls.host}:{simple_mockserver.port(is_secure=True)}/file"
@property
@abstractmethod
@@ -1317,6 +1328,9 @@ class TestHttpProxyBase(ABC):
assert response.body == self.expected_http_proxy_request_body
+PROXY_KINDS = ["http", "https", "socks5"]
+
+
class TestMitmProxyBase(ABC):
# whether the handler supports HTTPS proxies with HTTPS destinations
handler_supports_tls_in_tls: bool = True
@@ -1327,57 +1341,54 @@ class TestMitmProxyBase(ABC):
def settings_dict(self) -> dict[str, Any] | None:
raise NotImplementedError
- @pytest.mark.parametrize(
- "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
- )
- @pytest.mark.usefixtures("mitm_proxy_server")
- @coroutine_test
- async def test_http_proxy(
- self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
- ) -> None:
- """HTTP proxy, HTTP or HTTPS destination."""
- crawler = get_crawler(SingleRequestSpider, self.settings_dict)
- with caplog.at_level(logging.DEBUG):
- await crawler.crawl_async(
- seed=mockserver.url("/status?n=200", is_secure=https_dest)
- )
- assert isinstance(crawler.spider, SingleRequestSpider)
- self._assert_got_response_code(200, caplog.text)
- self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
-
- @pytest.mark.parametrize(
- "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
- )
- @pytest.mark.usefixtures("mitm_proxy_server_https")
- @coroutine_test
- async def test_https_proxy(
- self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
- ) -> None:
- """HTTPS proxy, HTTP or HTTPS destination."""
- if https_dest and not self.handler_supports_tls_in_tls:
+ def _maybe_skip(self, proxy_kind: str, https_dest: bool) -> None:
+ if proxy_kind == "socks5" and not self.handler_supports_socks:
+ pytest.skip("SOCKS proxies are not supported")
+ if (
+ proxy_kind == "https"
+ and https_dest
+ and not self.handler_supports_tls_in_tls
+ ):
pytest.skip("HTTPS proxies for HTTPS destinations are not supported")
- crawler = get_crawler(SingleRequestSpider, self.settings_dict)
- with caplog.at_level(logging.DEBUG):
- await crawler.crawl_async(
- seed=mockserver.url("/status?n=200", is_secure=https_dest)
- )
- assert isinstance(crawler.spider, SingleRequestSpider)
- self._assert_got_response_code(200, caplog.text)
- self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
+ @pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
- @pytest.mark.usefixtures("mitm_proxy_server")
@coroutine_test
- async def test_http_proxy_auth_error(
+ async def test_proxy(
self,
caplog: pytest.LogCaptureFixture,
- monkeypatch: pytest.MonkeyPatch,
+ proxy_server: str,
mockserver: MockServer,
https_dest: bool,
) -> None:
- """HTTP proxy, HTTP or HTTPS destination, wrong proxy creds."""
+ """HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination."""
+ self._maybe_skip(proxy_server, https_dest)
+ crawler = get_crawler(SingleRequestSpider, self.settings_dict)
+ with caplog.at_level(logging.DEBUG):
+ await crawler.crawl_async(
+ seed=mockserver.url("/status?n=200", is_secure=https_dest)
+ )
+ assert isinstance(crawler.spider, SingleRequestSpider)
+ self._assert_got_response_code(200, caplog.text)
+ self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
+
+ @pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True)
+ @pytest.mark.parametrize(
+ "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
+ )
+ @coroutine_test
+ async def test_proxy_auth_error(
+ self,
+ caplog: pytest.LogCaptureFixture,
+ monkeypatch: pytest.MonkeyPatch,
+ proxy_server: str,
+ mockserver: MockServer,
+ https_dest: bool,
+ ) -> None:
+ """HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination, wrong proxy creds."""
+ self._maybe_skip(proxy_server, https_dest)
envvar = "https_proxy" if https_dest else "http_proxy"
monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar]))
crawler = get_crawler(SimpleSpider, self.settings_dict)
@@ -1385,20 +1396,28 @@ class TestMitmProxyBase(ABC):
await crawler.crawl_async(
mockserver.url("/status?n=200", is_secure=https_dest)
)
- # The proxy returns a 407 error code but it does not reach the client;
- # it just sees an exception.
- self._assert_got_auth_exception(caplog.text)
+ if proxy_server == "socks5":
+ assert "DownloadConnectionRefusedError" in caplog.text
+ else:
+ # The proxy returns a 407 error code but it does not reach the
+ # client; it just sees an exception.
+ self._assert_got_auth_exception(caplog.text)
+ @pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
- @pytest.mark.usefixtures("mitm_proxy_server")
@coroutine_test
- async def test_dont_leak_proxy_authorization_header(
- self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
+ async def test_proxy_dont_leak_auth_header(
+ self,
+ caplog: pytest.LogCaptureFixture,
+ proxy_server: str,
+ mockserver: MockServer,
+ https_dest: bool,
) -> None:
- """HTTP proxy, HTTP or HTTPS destination. Check that the auth header
- is not sent to the destination."""
+ """HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination. Check that the
+ auth header is not sent to the destination."""
+ self._maybe_skip(proxy_server, https_dest)
request = Request(mockserver.url("/echo", is_secure=https_dest))
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
@@ -1409,48 +1428,36 @@ class TestMitmProxyBase(ABC):
echo = json.loads(crawler.spider.meta["responses"][0].text)
assert "Proxy-Authorization" not in echo["headers"]
+ @pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
- @pytest.mark.usefixtures("socks5_proxy_server")
@coroutine_test
- async def test_download_with_socks_proxy(
- self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
- ) -> None:
- """SOCKS5 proxy, HTTP or HTTPS destination."""
- if not self.handler_supports_socks:
- pytest.skip("SOCKS proxies are not supported")
- crawler = get_crawler(SingleRequestSpider, self.settings_dict)
- with caplog.at_level(logging.DEBUG):
- await crawler.crawl_async(
- seed=mockserver.url("/status?n=200", is_secure=https_dest)
- )
- assert isinstance(crawler.spider, SingleRequestSpider)
- self._assert_got_response_code(200, caplog.text)
- self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
-
- @pytest.mark.parametrize(
- "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
- )
- @pytest.mark.usefixtures("socks5_proxy_server")
- @coroutine_test
- async def test_socks_proxy_auth_error(
+ async def test_proxy_redirect(
self,
caplog: pytest.LogCaptureFixture,
- monkeypatch: pytest.MonkeyPatch,
+ proxy_server: str,
mockserver: MockServer,
https_dest: bool,
) -> None:
- if not self.handler_supports_socks:
- pytest.skip("SOCKS proxies are not supported")
- envvar = "https_proxy" if https_dest else "http_proxy"
- monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar]))
- crawler = get_crawler(SimpleSpider, self.settings_dict)
+ """HTTP/HTTPS/SOCKS5 proxy, HTTP or HTTPS destination, following a
+ redirect. Check that the redirected request still goes through the
+ proxy and doesn't lose the proxy auth.
+ """
+ self._maybe_skip(proxy_server, https_dest)
+ crawler = get_crawler(SingleRequestSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async(
- mockserver.url("/status?n=200", is_secure=https_dest)
+ seed=mockserver.url("/redirect", is_secure=https_dest)
)
- assert "DownloadConnectionRefusedError" in caplog.text
+ assert isinstance(crawler.spider, SingleRequestSpider)
+ assert crawler.spider.meta.get("failure") is None
+ responses = crawler.spider.meta.get("responses", [])
+ assert len(responses) == 1
+ assert responses[0].status == 200
+ assert responses[0].url == mockserver.url("/redirected", is_secure=https_dest)
+ self._assert_got_response_code(200, caplog.text)
+ self._assert_headers(responses[0].headers, https_dest)
@staticmethod
def _assert_headers(headers: Headers, https_dest: bool) -> None:
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 548c0d8ee..e5d726764 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -203,7 +203,7 @@ class DummyPolicyTestMixin(PolicyTestMixin):
assert mw.process_request(req) is None
# s3 scheme response is cached by default
- req, res = Request("s3://bucket/key"), Response("http://bucket/key")
+ req, res = Request("s3://bucket/key"), Response("s3://bucket/key")
with self._middleware() as mw:
assert mw.process_request(req) is None
mw.process_response(req, res)
@@ -214,7 +214,7 @@ class DummyPolicyTestMixin(PolicyTestMixin):
assert "cached" in cached.flags
# ignore s3 scheme
- req, res = Request("s3://bucket/key2"), Response("http://bucket/key2")
+ req, res = Request("s3://bucket/key2"), Response("s3://bucket/key2")
with self._middleware(HTTPCACHE_IGNORE_SCHEMES=["s3"]) as mw:
assert mw.process_request(req) is None
mw.process_response(req, res)
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index bb7fcd6c7..30caa094f 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -361,7 +361,7 @@ class TestHttpCompression:
zf.write(plainbody)
zf.close()
response = Response(
- "http;//www.example.com/", headers=headers, body=f.getvalue()
+ "http://www.example.com/", headers=headers, body=f.getvalue()
)
request = Request("http://www.example.com/")
@@ -386,7 +386,7 @@ class TestHttpCompression:
zf.write(plainbody)
zf.close()
response = HtmlResponse(
- "http;//www.example.com/page.html", headers=headers, body=f.getvalue()
+ "http://www.example.com/page.html", headers=headers, body=f.getvalue()
)
request = Request("http://www.example.com/")
@@ -493,7 +493,7 @@ class TestHttpCompression:
gz_resp.close()
response = Response(
- "http;//www.example.com/", headers=headers, body=r.getvalue()
+ "http://www.example.com/", headers=headers, body=r.getvalue()
)
request = Request("http://www.example.com/")
diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.py
index 44ade93b7..32935769f 100644
--- a/tests/test_downloadermiddleware_redirect_base.py
+++ b/tests/test_downloadermiddleware_redirect_base.py
@@ -122,7 +122,7 @@ class Base:
req1 = Request("http://a.example/first")
rsp1 = self.get_response(req1, "/redirected")
req2 = self.mw.process_response(req1, rsp1)
- rsp2 = self.get_response(req1, "/redirected2")
+ rsp2 = self.get_response(req2, "/redirected2")
req3 = self.mw.process_response(req2, rsp2)
assert req2.url == "http://a.example/redirected"
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 67af4264c..cf7b614c4 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -16,7 +16,7 @@ class TestDownloaderStats:
self.crawler.stats.open_spider()
self.req = Request("http://scrapytest.org")
- self.res = Response("scrapytest.org", status=400)
+ self.res = Response("http://scrapytest.org", status=400)
def assertStatsEqual(self, key, value):
assert self.crawler.stats.get_value(key) == value, str(
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index a717b18a6..9d58a6e09 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -5,7 +5,6 @@ import pytest
from scrapy import Request
from scrapy.core.downloader import Downloader, Slot
-from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
@@ -65,9 +64,6 @@ class TestCrawl:
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
- def setup_method(self):
- self.runner = CrawlerRunner()
-
@inline_callbacks_test
def test_delay(self):
crawler = get_crawler(DownloaderSlotsSettingsTestSpider)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index b38bf9570..412a59fcd 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -86,7 +86,7 @@ class TestRFPDupeFilter:
df.close("finished")
df2 = _get_dupefilter(settings={"JOBDIR": path}, open_=False)
- assert df != df2
+ assert df is not df2
try:
df2.open()
assert df2.request_seen(r1)
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 2cd583721..e51eb4664 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -6,7 +6,6 @@ import subprocess
import sys
from collections import defaultdict
from dataclasses import dataclass
-from logging import DEBUG
from typing import TYPE_CHECKING, Any, cast
from unittest.mock import Mock, call
from urllib.parse import urlparse
@@ -395,6 +394,7 @@ class TestEngine(TestEngineBase):
self._assert_downloaded_responses(run, count=9)
self._assert_scraped_items(run)
self._assert_signals_caught(run)
+ self._assert_headers_received(run)
self._assert_bytes_received(run)
@coroutine_test
@@ -606,7 +606,7 @@ class TestEngineDownload(TestEngineDownloadAsync):
@coroutine_test
-async def test_request_scheduled_signal(caplog):
+async def test_request_scheduled_signal():
class TestScheduler(BaseScheduler):
def __init__(self):
self.enqueued = []
@@ -633,7 +633,6 @@ async def test_request_scheduled_signal(caplog):
keep_request = Request("https://keep.example")
engine._schedule_request(keep_request)
drop_request = Request("https://drop.example")
- caplog.set_level(DEBUG)
engine._schedule_request(drop_request)
assert scheduler.enqueued == [keep_request], (
f"{scheduler.enqueued!r} != [{keep_request!r}]"
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 27e0c6445..c1d6f04eb 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -32,7 +32,6 @@ from scrapy.extensions.feedexport import (
FeedSlot,
FileFeedStorage,
IFeedStorage,
- S3FeedStorage,
)
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
@@ -499,8 +498,7 @@ class TestFeedExport(TestFeedExportBase):
},
}
crawler = get_crawler(ItemSpider, settings)
- with mock.patch.object(S3FeedStorage, "store"):
- yield crawler.crawl(mockserver=self.mockserver)
+ yield crawler.crawl(mockserver=self.mockserver)
assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
assert "feedexport/success_count/StdoutFeedStorage" in crawler.stats.get_stats()
assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 1
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index d855d0f74..0a926479b 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -315,7 +315,7 @@ class TestBatchDeliveries(TestFeedExportBase):
}
data = await self.exported_data(items, settings)
for fmt, expected in formats.items():
- for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
+ for expected_batch, got_batch in zip(expected, data[fmt], strict=True):
assert got_batch == expected_batch
@coroutine_test
@@ -339,7 +339,7 @@ class TestBatchDeliveries(TestFeedExportBase):
}
data = await self.exported_data(items, settings)
for fmt, expected in formats.items():
- for expected_batch, got_batch in zip(expected, data[fmt], strict=False):
+ for expected_batch, got_batch in zip(expected, data[fmt], strict=True):
assert got_batch == expected_batch
@coroutine_test
@@ -447,7 +447,7 @@ class TestBatchDeliveries(TestFeedExportBase):
yield crawler.crawl()
assert len(CustomS3FeedStorage.stubs) == len(items)
- for stub in CustomS3FeedStorage.stubs[:-1]:
+ for stub in CustomS3FeedStorage.stubs:
stub.assert_no_pending_responses()
assert (
"feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats()
diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py
index fa1c0586a..6ebcab152 100644
--- a/tests/test_feedexport_postprocess.py
+++ b/tests/test_feedexport_postprocess.py
@@ -270,7 +270,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
self._named_tempfile("check_CHECK_NONE"): lzma.compress(
self.expected, check=lzma.CHECK_NONE
),
- self._named_tempfile("check_CHECK_CRC256"): lzma.compress(
+ self._named_tempfile("CHECK_SHA256"): lzma.compress(
self.expected, check=lzma.CHECK_SHA256
),
}
@@ -282,7 +282,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_check": lzma.CHECK_NONE,
},
- self._named_tempfile("check_CHECK_CRC256"): {
+ self._named_tempfile("CHECK_SHA256"): {
"format": "csv",
"postprocessing": ["scrapy.extensions.postprocessing.LZMAPlugin"],
"lzma_check": lzma.CHECK_SHA256,
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index cec5d728b..28f306e31 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -141,7 +141,7 @@ class Dataloss(LeafResource):
class NoContentLengthHeader(LeafResource):
def render_GET(self, request: TxRequest):
- request.requestHeaders.removeHeader("Content-Length")
+ request.responseHeaders.removeHeader("Content-Length")
self.deferRequest(request, 0, self._delayed_render, request)
return NOT_DONE_YET
@@ -460,9 +460,7 @@ class TestHttps2ClientProtocol:
def test_invalid_negotiated_protocol(
self, server_port: int, client: H2ClientProtocol
) -> Generator[Deferred[Any], Any, None]:
- with mock.patch(
- "scrapy.core.http2.protocol.PROTOCOL_NAME", return_value=b"not-h2"
- ):
+ with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", new=b"not-h2"):
request = Request(url=self.get_url(server_port, "/status?n=200"))
with pytest.raises(ResponseFailed):
yield make_request_dfd(client, request)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index fed5dbab7..fd494504d 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -23,7 +23,6 @@ class TestRequest:
# url argument must be basestring
with pytest.raises(TypeError):
self.request_class(123)
- r = self.request_class("http://www.example.com")
r = self.request_class("http://www.example.com")
assert isinstance(r.url, str)
@@ -211,11 +210,11 @@ class TestRequest:
r1.cb_kwargs["key"] = "value"
r2 = r1.copy()
- # make sure copy does not propagate callbacks
+ # make sure callbaclks are copied
assert r1.callback is somecallback
assert r1.errback is somecallback
assert r2.callback is r1.callback
- assert r2.errback is r2.errback
+ assert r2.errback is r1.errback
# make sure flags list is shallow copied
assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 09c95dc29..079c547c7 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,13 +1,19 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
import pytest
-from packaging.version import Version as parse_version
-from w3lib import __version__ as w3lib_version
from w3lib.encoding import resolve_encoding
from scrapy.exceptions import NotSupported
from scrapy.http import Headers, Request, Response
from scrapy.link import Link
+from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS
from tests import get_testdata
+if TYPE_CHECKING:
+ from collections.abc import Iterable
+
class TestResponse:
response_class = Response
@@ -249,7 +255,7 @@ class TestResponse:
r.follow(None)
@pytest.mark.xfail(
- parse_version(w3lib_version) < parse_version("2.1.1"),
+ not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
)
@@ -257,7 +263,7 @@ class TestResponse:
self._assert_followed_url("foo ", "http://example.com/foo")
@pytest.mark.xfail(
- parse_version(w3lib_version) < parse_version("2.1.1"),
+ not W3LIB_STRIPS_URLS,
reason="https://github.com/scrapy/w3lib/pull/207",
strict=True,
)
@@ -325,16 +331,26 @@ class TestResponse:
with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
+ @pytest.mark.xfail(
+ not W3LIB_STRIPS_URLS,
+ reason="https://github.com/scrapy/w3lib/pull/207",
+ strict=True,
+ )
def test_follow_all_whitespace(self):
relative = ["foo ", "bar ", "foo/bar ", "bar/foo "]
absolute = [
- "http://example.com/foo%20",
- "http://example.com/bar%20",
- "http://example.com/foo/bar%20",
- "http://example.com/bar/foo%20",
+ "http://example.com/foo",
+ "http://example.com/bar",
+ "http://example.com/foo/bar",
+ "http://example.com/bar/foo",
]
self._assert_followed_all_urls(relative, absolute)
+ @pytest.mark.xfail(
+ not W3LIB_STRIPS_URLS,
+ reason="https://github.com/scrapy/w3lib/pull/207",
+ strict=True,
+ )
def test_follow_all_whitespace_links(self):
absolute = [
"http://example.com/foo ",
@@ -342,8 +358,8 @@ class TestResponse:
"http://example.com/foo/bar ",
"http://example.com/bar/foo ",
]
- links = map(Link, absolute)
- expected = [u.replace(" ", "%20") for u in absolute]
+ links = [Link(u) for u in absolute]
+ expected = [u.strip() for u in absolute]
self._assert_followed_all_urls(links, expected)
def test_follow_all_flags(self):
@@ -357,25 +373,36 @@ class TestResponse:
for req in fol:
assert req.flags == ["cached", "allowed"]
- def _assert_followed_url(self, follow_obj, target_url, response=None):
+ def _assert_followed_url(
+ self,
+ follow_obj: str | Link,
+ target_url: str,
+ response: Response | None = None,
+ encoding: str | None = None,
+ ) -> None:
if response is None:
response = self._links_response()
req = response.follow(follow_obj)
assert req.url == target_url
- return req
+ if encoding is not None:
+ assert req.encoding == encoding
- def _assert_followed_all_urls(self, follow_obj, target_urls, response=None):
+ def _assert_followed_all_urls(
+ self,
+ follow_obj: Iterable[str | Link],
+ target_urls: Iterable[str],
+ response: Response | None = None,
+ ) -> None:
if response is None:
response = self._links_response()
followed = response.follow_all(follow_obj)
- for req, target in zip(followed, target_urls, strict=False):
+ for req, target in zip(followed, target_urls, strict=True):
assert req.url == target
- yield req
- def _links_response(self):
+ def _links_response(self) -> Response:
body = get_testdata("link_extractor", "linkextractor.html")
return self.response_class("http://example.com/index", body=body)
- def _links_response_no_href(self):
+ def _links_response_no_href(self) -> Response:
body = get_testdata("link_extractor", "linkextractor_no_href.html")
return self.response_class("http://example.com/index", body=body)
diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index c16af52b9..4b3fa2302 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -179,7 +179,6 @@ class TestTextResponse(TestResponse):
# Inferring encoding from body also cache decoded body as sideeffect,
# this test tries to ensure that calling response.encoding and
# response.text in indistinct order doesn't affect final
- # response.text in indistinct order doesn't affect final
# values for encoding and decoded body.
url = "http://example.com"
body = b"\xef\xbb\xbfWORD"
@@ -308,11 +307,12 @@ class TestTextResponse(TestResponse):
"http://example.com/sample3.html#foo",
"http://www.google.com/something",
"http://example.com/innertag.html",
+ "http://example.com/page%204.html",
]
# select elements
for sellist in [resp.css("a"), resp.xpath("//a")]:
- for sel, url in zip(sellist, urls, strict=False):
+ for sel, url in zip(sellist, urls, strict=True):
self._assert_followed_url(sel, url, response=resp)
# select elements
@@ -324,7 +324,7 @@ class TestTextResponse(TestResponse):
# href attributes should work
for sellist in [resp.css("a::attr(href)"), resp.xpath("//a/@href")]:
- for sel, url in zip(sellist, urls, strict=False):
+ for sel, url in zip(sellist, urls, strict=True):
self._assert_followed_url(sel, url, response=resp)
# non-a elements are not supported
@@ -376,12 +376,12 @@ class TestTextResponse(TestResponse):
encoding="utf8",
body='click me'.encode(),
)
- req = self._assert_followed_url(
+ self._assert_followed_url(
resp1.css("a")[0],
"http://example.com/foo?%D0%BF%D1%80%D0%B8%D0%B2%D0%B5%D1%82",
response=resp1,
+ encoding="utf8",
)
- assert req.encoding == "utf8"
resp2 = self.response_class(
"http://example.com",
@@ -390,12 +390,12 @@ class TestTextResponse(TestResponse):
"cp1251"
),
)
- req = self._assert_followed_url(
+ self._assert_followed_url(
resp2.css("a")[0],
"http://example.com/foo?%EF%F0%E8%E2%E5%F2",
response=resp2,
+ encoding="cp1251",
)
- assert req.encoding == "cp1251"
def test_follow_flags(self):
res = self.response_class("http://example.com/")
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 9806315b4..360aa613e 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -28,14 +28,14 @@ class TestLogFormatter:
self.spider = Spider("default")
self.spider.crawler = get_crawler()
- def test_crawled_with_referer(self):
+ def test_crawled_without_referer(self):
req = Request("http://www.example.com")
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws["msg"] % logkws["args"]
assert logline == "Crawled (200) (referer: None)"
- def test_crawled_without_referer(self):
+ def test_crawled_with_referer(self):
req = Request(
"http://www.example.com", headers={"referer": "http://example.com"}
)
@@ -198,7 +198,7 @@ class TestLogformatterSubclass(TestLogFormatter):
self.spider = Spider("default")
self.spider.crawler = get_crawler(Spider)
- def test_crawled_with_referer(self):
+ def test_crawled_without_referer(self):
req = Request("http://www.example.com")
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
@@ -207,7 +207,7 @@ class TestLogformatterSubclass(TestLogFormatter):
logline == "Crawled (200) (referer: None) []"
)
- def test_crawled_without_referer(self):
+ def test_crawled_with_referer(self):
req = Request(
"http://www.example.com",
headers={"referer": "http://example.com"},
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 38662348f..1b73dd157 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -14,6 +14,7 @@ from itemadapter import ItemAdapter
from scrapy.http import Request, Response
from scrapy.item import Field, Item
+from scrapy.pipelines.files import GCSFilesStore, S3FilesStore
from scrapy.pipelines.images import ImageException, ImagesPipeline
from scrapy.utils.test import get_crawler
@@ -541,6 +542,44 @@ class TestImagesPipelineCustomSettings:
expected_value = settings.get(settings_attr)
assert getattr(pipeline_cls, pipe_attr.lower()) == expected_value
+ def test_images_store_s3_acl_setting_used(self, tmp_path):
+ old_policy = S3FilesStore.POLICY
+
+ try:
+ crawler = get_crawler(
+ None,
+ {
+ "IMAGES_STORE": tmp_path,
+ "IMAGES_STORE_S3_ACL": "public-read",
+ "FILES_STORE_S3_ACL": "private",
+ },
+ )
+
+ ImagesPipeline.from_crawler(crawler)
+
+ assert S3FilesStore.POLICY == "public-read"
+ finally:
+ S3FilesStore.POLICY = old_policy
+
+ def test_images_store_gcs_acl_setting_used(self, tmp_path):
+ old_policy = GCSFilesStore.POLICY
+
+ try:
+ crawler = get_crawler(
+ None,
+ {
+ "IMAGES_STORE": tmp_path,
+ "IMAGES_STORE_GCS_ACL": "authenticatedRead",
+ "FILES_STORE_GCS_ACL": "",
+ },
+ )
+
+ ImagesPipeline.from_crawler(crawler)
+
+ assert GCSFilesStore.POLICY == "authenticatedRead"
+ finally:
+ GCSFilesStore.POLICY = old_policy
+
def _create_image(format_, *a, **kw):
buf = io.BytesIO()
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 44df0bdd4..da1bfa317 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -175,10 +175,6 @@ class MockedMediaPipeline(UserDefinedPipeline):
super().__init__(*args, crawler=crawler, **kwargs)
self._mockcalled = []
- def download(self, request, info):
- self._mockcalled.append("download")
- return super().download(request, info)
-
def media_to_download(self, request, info, *, item=None):
self._mockcalled.append("media_to_download")
if "result" in request.meta:
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 29b23496a..5249736f2 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,5 +1,3 @@
-import sys
-
import pytest
from scrapy.robotstxt import (
@@ -8,6 +6,7 @@ from scrapy.robotstxt import (
RerpRobotParser,
decode_robotstxt,
)
+from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
def rerp_available() -> bool:
@@ -139,28 +138,25 @@ class TestDecodeRobotsTxt:
class TestPythonRobotParser(BaseRobotParserTest):
- # https://github.com/python/cpython/pull/149374 improves it
- IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5)
-
def setup_method(self):
super()._setUp(PythonRobotParser)
@pytest.mark.skipif(
- not IMPROVED_ROBOTFILEPARSER,
+ not STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support length based directives precedence.",
)
def test_length_based_precedence(self):
super().test_length_based_precedence()
@pytest.mark.skipif(
- IMPROVED_ROBOTFILEPARSER,
+ STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support order based directives precedence.",
)
def test_order_based_precedence(self):
super().test_order_based_precedence()
@pytest.mark.skipif(
- not IMPROVED_ROBOTFILEPARSER,
+ not STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support wildcards.",
)
def test_allowed_wildcards(self):
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
index db023e1f8..08acacae7 100644
--- a/tests/test_scheduler_base.py
+++ b/tests/test_scheduler_base.py
@@ -104,7 +104,7 @@ class TestMinimalScheduler(InterfaceCheckMixin):
for url in URLS:
assert self.scheduler.enqueue_request(Request(url))
assert not self.scheduler.enqueue_request(Request(url))
- assert self.scheduler.has_pending_requests
+ assert self.scheduler.has_pending_requests()
dequeued = []
while self.scheduler.has_pending_requests():
diff --git a/tests/test_signalmanager.py b/tests/test_signalmanager.py
new file mode 100644
index 000000000..ce4d97adb
--- /dev/null
+++ b/tests/test_signalmanager.py
@@ -0,0 +1,21 @@
+from scrapy.signalmanager import SignalManager
+
+
+class TestSignalManager:
+ def test_disconnect_all(self):
+ signal = object()
+ sender = object()
+ sm = SignalManager(sender)
+
+ calls = []
+
+ def handler():
+ calls.append(1)
+
+ sm.connect(handler, signal)
+ sm.send_catch_log(signal)
+ assert calls == [1]
+
+ sm.disconnect_all(signal)
+ sm.send_catch_log(signal)
+ assert calls == [1] # handler no longer called after disconnect_all
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index a0d296553..e5891474b 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -238,16 +238,6 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
yield item
-class ProcessSpiderOutputNonIterableMiddleware:
- def process_spider_output(self, response, result):
- return
-
-
-class ProcessSpiderOutputCoroutineMiddleware:
- async def process_spider_output(self, response, result):
- return result
-
-
class ProcessStartSimpleMiddleware:
async def process_start(self, start):
async for item_or_request in start:
@@ -423,20 +413,12 @@ class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
ITEM_TYPE = dict
MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
- MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware
MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIteratorMiddleware
def _callback(self) -> Any:
1 / 0
- async def _test_asyncgen_nodowngrade(self, *mw_classes: type[Any]) -> None:
- with pytest.raises(
- _InvalidOutput,
- match=r"Async iterable returned from .+ cannot be downgraded",
- ):
- await self._get_middleware_result(*mw_classes)
-
@coroutine_test
async def test_exc_simple(self):
"""Simple exc mw"""
diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py
index c907c6d73..21df73a65 100644
--- a/tests/test_spidermiddleware_process_start.py
+++ b/tests/test_spidermiddleware_process_start.py
@@ -14,7 +14,6 @@ from .utils.decorators import coroutine_test
ITEM_A = {"id": "a"}
ITEM_B = {"id": "b"}
ITEM_C = {"id": "c"}
-ITEM_D = {"id": "d"}
class AsyncioSleepSpiderMiddleware:
@@ -47,10 +46,6 @@ class ModernWrapSpider(Spider):
yield ITEM_B
-class ModernWrapSpiderSubclass(ModernWrapSpider):
- name = "test"
-
-
class ModernWrapSpiderMiddleware:
async def process_start(self, start):
yield ITEM_A
@@ -79,10 +74,6 @@ class TestMain:
expected_items = expected_items or [ITEM_A, ITEM_B, ITEM_C]
await self._test([spider_middleware], spider_cls, expected_items)
- async def _test_douple_wrap(self, smw1, smw2, spider_cls, expected_items=None):
- expected_items = expected_items or [ITEM_A, ITEM_A, ITEM_B, ITEM_C, ITEM_C]
- await self._test([smw1, smw2], spider_cls, expected_items)
-
@coroutine_test
async def test_modern_mw_modern_spider(self):
with warnings.catch_warnings():
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 7431ea6ac..a9089419a 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -836,7 +836,7 @@ class TestRequestMetaSettingFallback:
request_meta,
policy_class,
check_warning,
- ) in self.params[3:]:
+ ) in self.params:
mw = RefererMiddleware(Settings(settings))
response = Response(origin, headers=response_headers)
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index 491fc88f7..e44cfca90 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -1,4 +1,7 @@
+from __future__ import annotations
+
from datetime import datetime, timezone
+from typing import TYPE_CHECKING
import pytest
@@ -7,8 +10,11 @@ from scrapy.extensions.spiderstate import SpiderState
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
+if TYPE_CHECKING:
+ from pathlib import Path
-def test_store_load(tmp_path):
+
+def test_store_load(tmp_path: Path) -> None:
jobdir = str(tmp_path)
spider = Spider(name="default")
@@ -16,6 +22,7 @@ def test_store_load(tmp_path):
ss = SpiderState(jobdir)
ss.spider_opened(spider)
+ assert hasattr(spider, "state")
spider.state["one"] = 1
spider.state["dt"] = dt
ss.spider_closed(spider)
@@ -23,21 +30,23 @@ def test_store_load(tmp_path):
spider2 = Spider(name="default")
ss2 = SpiderState(jobdir)
ss2.spider_opened(spider2)
- assert spider.state == {"one": 1, "dt": dt}
+ assert hasattr(spider2, "state")
+ assert spider2.state == {"one": 1, "dt": dt}
ss2.spider_closed(spider2)
-def test_state_attribute():
+def test_state_attribute() -> None:
# state attribute must be present if jobdir is not set, to provide a
# consistent interface
spider = Spider(name="default")
ss = SpiderState()
ss.spider_opened(spider)
+ assert hasattr(spider, "state")
assert spider.state == {}
ss.spider_closed(spider)
-def test_not_configured():
+def test_not_configured() -> None:
crawler = get_crawler(Spider)
with pytest.raises(NotConfigured):
SpiderState.from_crawler(crawler)
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index a871a282e..5532b4a31 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -83,6 +83,7 @@ class TestParallelAsyncio:
max_parallel_count,
)
assert list(range(length)) == sorted(results)
+ assert parallel_count[0] == 0
assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
@coroutine_test
@@ -101,6 +102,7 @@ class TestParallelAsyncio:
max_parallel_count,
)
assert list(range(length)) == sorted(results)
+ assert parallel_count[0] == 0
assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index c5425d99d..5ea6f678e 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -22,9 +22,7 @@ class NewName(SomeBaseClass):
class TestWarnWhenSubclassed:
- def _mywarnings(
- self, w: list[WarningMessage], category: type[Warning] = MyWarning
- ) -> list[WarningMessage]:
+ def _mywarnings(self, w: list[WarningMessage]) -> list[WarningMessage]:
return [x for x in w if x.category is MyWarning]
def test_no_warning_on_definition(self):
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index 9bd86f7fb..0eb330461 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -17,4 +17,4 @@ def test_urlparse_cached():
assert req1a == urlp
assert req1a is req1b
assert req1a is not req2
- assert req1a is not req2
+ assert req1b is not req2
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 8e5020022..ee552df64 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -76,15 +76,16 @@ class TestLogCounterHandler:
@pytest.fixture
def logger(self, crawler: Crawler) -> Generator[logging.Logger]:
logger = logging.getLogger("test")
- logger.setLevel(logging.NOTSET)
+ logger.setLevel(logging.DEBUG)
logger.propagate = False
- handler = LogCounterHandler(crawler)
+ handler = LogCounterHandler(crawler, level=crawler.settings.get("LOG_LEVEL"))
logger.addHandler(handler)
-
- yield logger
-
- logger.propagate = True
- logger.removeHandler(handler)
+ try:
+ yield logger
+ finally:
+ logger.propagate = True
+ logger.setLevel(logging.NOTSET)
+ logger.removeHandler(handler)
def test_init(self, crawler: Crawler, logger: logging.Logger) -> None:
assert crawler.stats
@@ -102,7 +103,7 @@ class TestLogCounterHandler:
def test_filtered_out_level(self, crawler: Crawler, logger: logging.Logger) -> None:
logger.debug("test log msg")
assert crawler.stats
- assert crawler.stats.get_value("log_count/INFO") is None
+ assert crawler.stats.get_value("log_count/DEBUG") is None
class TestStreamLogger:
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index a995e38e6..c4c861404 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -160,3 +160,8 @@ class TestUtilsMisc:
assert rel_has_nofollow("nofollowfoo") is False
assert rel_has_nofollow("foonofollow") is False
assert rel_has_nofollow("ugc, , nofollow") is True
+ # rel attribute values are ASCII case-insensitive per the HTML spec
+ assert rel_has_nofollow("NoFollow") is True
+ assert rel_has_nofollow("NOFOLLOW") is True
+ assert rel_has_nofollow("UGC NoFollow") is True
+ assert rel_has_nofollow("ugc,NoFollow") is True
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 55c46059e..e4967d4e7 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -413,3 +413,45 @@ class TestRequestToCurl:
" --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'"
)
self._test_request(request_object, expected_curl_command)
+
+ def test_cookies_list_verbose(self):
+ request_object = Request(
+ "https://www.httpbin.org/post",
+ method="POST",
+ cookies=[
+ {
+ "name": b"foo",
+ "value": b"bar",
+ "domain": "example.com",
+ "path": "/",
+ "secure": True,
+ }
+ ],
+ body=json.dumps({"foo": "bar"}),
+ )
+ expected_curl_command = (
+ "curl -X POST https://www.httpbin.org/post"
+ " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=bar'"
+ )
+ self._test_request(request_object, expected_curl_command)
+
+ def test_cookies_list_verbose_non_string_value(self):
+ request_object = Request(
+ "https://www.httpbin.org/post",
+ method="POST",
+ cookies=[
+ {
+ "name": "foo",
+ "value": 1,
+ "domain": "example.com",
+ "path": "/",
+ "secure": True,
+ }
+ ],
+ body=json.dumps({"foo": "bar"}),
+ )
+ expected_curl_command = (
+ "curl -X POST https://www.httpbin.org/post"
+ " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=1'"
+ )
+ self._test_request(request_object, expected_curl_command)
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index e02bdfb69..4544cd29e 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -23,7 +23,7 @@ def _read_browser_output(burl: str):
def test_open_in_browser():
- url = "http:///www.example.com/some/page.html"
+ url = "http://www.example.com/some/page.html"
body = (
b" test page test body "
)
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 19a31c353..a74b9a41d 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -346,6 +346,18 @@ class TestStripUrl:
"ftp://username:password@www.example.com:221/file.txt",
"ftp://username:password@www.example.com:221/file.txt",
),
+ (
+ "http://user:80@www.example.com:80/index.html",
+ "http://user:80@www.example.com/index.html",
+ ),
+ (
+ "https://user:443@www.example.com:443/index.html",
+ "https://user:443@www.example.com/index.html",
+ ),
+ (
+ "ftp://user:21@www.example.com:21/file.txt",
+ "ftp://user:21@www.example.com/file.txt",
+ ),
],
)
def test_default_ports(self, url: str, expected: str) -> None:
diff --git a/tox.ini b/tox.ini
index fddb0d1cb..aa20b8e83 100644
--- a/tox.ini
+++ b/tox.ini
@@ -89,7 +89,7 @@ deps =
ptpython==3.0.32
# newer ones require newer Python
ipython==8.39.0
- pyOpenSSL==26.2.0
+ pyOpenSSL==26.3.0
pytest==9.0.3
socksio==1.0.0
types-Pygments==2.20.0.20260508