From f38c2afe912fd16f85bd9efaea9fcf6ad22a7f29 Mon Sep 17 00:00:00 2001 From: Diogo Castro Date: Mon, 22 Jun 2026 10:10:02 -0300 Subject: [PATCH] fix: Adding protocol_relative_pattern to reject another pattern for invalid URLs --- scrapy/downloadermiddlewares/offsite.py | 3 ++- tests/test_downloadermiddleware_offsite.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index fc9f35a66..5dcd67bc7 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -101,13 +101,14 @@ class OffsiteMiddleware: misconfigured domains. """ url_pattern = re.compile(r"^https?://.*$") + protocol_relative_pattern = re.compile(r"^//") port_pattern = re.compile(r":\d+$") valid_domains: list[str] = [] for domain in domains_list: if domain is None: raise ValueError(f"{domains_type} contains empty value.") - if url_pattern.match(domain): + if url_pattern.match(domain) or protocol_relative_pattern.match(domain): raise ValueError( f"{domains_type} accepts only domains, not URLs. " f"Got URL entry {domain} in {domains_type}." diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index 20172c02c..62e236fdf 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -121,6 +121,7 @@ def test_process_request_no_allowed_domains(value): [ ["a.example", None], ["a.example", "http://b.example"], + ["a.example", "//c.example"], ["a.example", "c.example:8080"], ], ) @@ -233,6 +234,7 @@ def test_request_scheduled_no_allowed_domains(value): [ ["a.example", None], ["a.example", "http://b.example"], + ["a.example", "//c.example"], ["a.example", "c.example:8080"], ], )