From 9d2dea7a8df4f1c2cb59b71bca143148a5306c36 Mon Sep 17 00:00:00 2001 From: Adrian Date: Sun, 9 Aug 2026 10:57:44 +0200 Subject: [PATCH] Stop skipping the IPv6 resolver tests (#7935) --- .../AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py | 5 ++++- tests/AsyncCrawlerProcess/default_name_resolver.py | 5 ++++- tests/CrawlerProcess/caching_hostname_resolver_ipv6.py | 5 ++++- tests/CrawlerProcess/default_name_resolver.py | 5 ++++- tests/test_crawler_subprocess.py | 7 +------ 5 files changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py index 55d2ef711..8181c4d17 100644 --- a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py @@ -8,7 +8,10 @@ class CachingHostnameResolverSpider(scrapy.Spider): """ name = "caching_hostname_resolver_spider" - start_urls = ["http://[::1]"] + + async def start(self): + # w3lib older than 2.4.1 strips the brackets, making the URL invalid. + yield scrapy.Request("http://[::1]", meta={"verbatim_url": True}) if __name__ == "__main__": diff --git a/tests/AsyncCrawlerProcess/default_name_resolver.py b/tests/AsyncCrawlerProcess/default_name_resolver.py index 4c8897f8f..7cc59594b 100644 --- a/tests/AsyncCrawlerProcess/default_name_resolver.py +++ b/tests/AsyncCrawlerProcess/default_name_resolver.py @@ -9,7 +9,10 @@ class IPv6Spider(scrapy.Spider): """ name = "ipv6_spider" - start_urls = ["http://[::1]"] + + async def start(self): + # w3lib older than 2.4.1 strips the brackets, making the URL invalid. + yield scrapy.Request("http://[::1]", meta={"verbatim_url": True}) if __name__ == "__main__": diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py index da9c16cb8..f6f865e3e 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py +++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py @@ -8,7 +8,10 @@ class CachingHostnameResolverSpider(scrapy.Spider): """ name = "caching_hostname_resolver_spider" - start_urls = ["http://[::1]"] + + async def start(self): + # w3lib older than 2.4.1 strips the brackets, making the URL invalid. + yield scrapy.Request("http://[::1]", meta={"verbatim_url": True}) if __name__ == "__main__": diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py index f4c129fdf..12b894030 100644 --- a/tests/CrawlerProcess/default_name_resolver.py +++ b/tests/CrawlerProcess/default_name_resolver.py @@ -9,7 +9,10 @@ class IPv6Spider(scrapy.Spider): """ name = "ipv6_spider" - start_urls = ["http://[::1]"] + + async def start(self): + # w3lib older than 2.4.1 strips the brackets, making the URL invalid. + yield scrapy.Request("http://[::1]", meta={"verbatim_url": True}) if __name__ == "__main__": diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index 733b6797d..240482586 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -10,9 +10,7 @@ from pathlib import Path from typing import TYPE_CHECKING import pytest -from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn -from w3lib import __version__ as w3lib_version from scrapy.utils.asyncio import sleep from tests.utils import get_script_run_env @@ -97,10 +95,6 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): ) assert "RuntimeError" not in log - @pytest.mark.skipif( - parse_version(w3lib_version) >= parse_version("2.0.0"), - reason="w3lib 2.0.0 and later do not allow invalid domains.", - ) def test_ipv6_default_name_resolver(self) -> None: log = self.run_script("default_name_resolver.py") assert "Spider closed (finished)" in log @@ -116,6 +110,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): def test_caching_hostname_resolver_ipv6(self) -> None: log = self.run_script("caching_hostname_resolver_ipv6.py") assert "Spider closed (finished)" in log + assert "http://::1" not in log assert "scrapy.exceptions.CannotResolveHostError" not in log def test_caching_hostname_resolver_finite_execution(