From 92b5312374323585c7f73098deee4fadfaca38b8 Mon Sep 17 00:00:00 2001
From: Diogo Castro <diogo.neves.castro@gmail.com>
Date: Sat, 17 May 2025 12:26:58 -0300
Subject: [PATCH 01/13] feat: Adding basic disallowed_domains functionality

---
 scrapy/downloadermiddlewares/offsite.py | 90 ++++++++++++++++++-------
 1 file changed, 67 insertions(+), 23 deletions(-)

diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index 33d7ba609..335007d8d 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -71,29 +71,73 @@ class OffsiteMiddleware:
         return bool(regex.search(host))
 
     def get_host_regex(self, spider: Spider) -> re.Pattern[str]:
-        """Override this method to implement a different offsite policy"""
-        allowed_domains = getattr(spider, "allowed_domains", None)
-        if not allowed_domains:
+        """Override this method to implement a different offsite policy.
+        Returns a compiled regular expression object that matches the hosts that
+        are allowed to be crawled. If None is returned (or method is not overridden),
+        all hosts are allowed.
+        Example:
+        allowed_domains = ['example.com']
+        disallowed_domains = ['example2.com']
+        This will allow crawling all subdomains of example.com (eg. foo.example.com,
+        bar.example.com). But it won't allow crawling example2.com or any subdomain
+        (eg. www.example2.com).
+        """
+        allowed_domains_arg = getattr(spider, "allowed_domains", None)
+        disallowed_domains_arg = getattr(spider, "disallowed_domains", None)
+        allowed_domains = []
+        disallowed_domains = []
+        if not allowed_domains_arg and not disallowed_domains_arg:
             return re.compile("")  # allow all by default
+
         url_pattern = re.compile(r"^https?://.*$")
         port_pattern = re.compile(r":\d+$")
-        domains = []
-        for domain in allowed_domains:
-            if domain is None:
-                continue
-            if url_pattern.match(domain):
-                message = (
-                    "allowed_domains accepts only domains, not URLs. "
-                    f"Ignoring URL entry {domain} in allowed_domains."
-                )
-                warnings.warn(message)
-            elif port_pattern.search(domain):
-                message = (
-                    "allowed_domains accepts only domains without ports. "
-                    f"Ignoring entry {domain} in allowed_domains."
-                )
-                warnings.warn(message)
-            else:
-                domains.append(re.escape(domain))
-        regex = rf"^(.*\.)?({'|'.join(domains)})$"
-        return re.compile(regex)
+        # domains = []
+
+        def process_domains(domains_list=[], domains_type="allowed_domains"):
+            """
+            Process the domains list and return a list of valid domains.
+            The arguments passed to the spider in allowed_domains and disallowed_domains
+            cannot be URLs or contain ports.
+            """
+            valid_domains = []
+
+            for domain in domains_list:
+                if domain is None:
+                    continue
+                if url_pattern.match(domain):
+                    message = (
+                        f"{domains_type} accepts only domains, not URLs. "
+                        f"Ignoring URL entry {domain} in {domains_type}."
+                    )
+                    warnings.warn(message)
+                elif port_pattern.search(domain):
+                    message = (
+                        f"{domains_type} accepts only domains without ports. "
+                        f"Ignoring entry {domain} in {domains_type}."
+                    )
+                    warnings.warn(message)
+                else:
+                    valid_domains.append(re.escape(domain))
+            return valid_domains
+
+        if allowed_domains_arg:
+            allowed_domains = process_domains(allowed_domains_arg, "allowed_domains")
+
+        if disallowed_domains_arg:
+            disallowed_domains = process_domains(disallowed_domains_arg, "disallowed_domains")
+
+        if allowed_domains:
+            allowed_domain_pattern = rf"^(.*\.)?({'|'.join(allowed_domains)})$"
+        else:
+            allowed_domain_pattern = ""
+
+        if disallowed_domains:
+            disallowed_domain_pattern = rf"^(?!.*(?:{'|'.join(disallowed_domains)}))$"
+        else:
+            disallowed_domain_pattern = ""
+
+        if allowed_domain_pattern and disallowed_domain_pattern:
+            combined_pattern = rf"{allowed_domain_pattern}|{disallowed_domain_pattern}"
+        else:
+            combined_pattern = allowed_domain_pattern or disallowed_domain_pattern
+        return re.compile(combined_pattern)

From bac921801a427a6baff433ba9a4dd36af00a0c0d Mon Sep 17 00:00:00 2001
From: Diogo Castro <diogo.castro@zyte.com>
Date: Fri, 19 Sep 2025 09:28:49 -0300
Subject: [PATCH 02/13] feat: Starting to add new tests to check for
 disallowed_domains feature

---
 scrapy/downloadermiddlewares/offsite.py    |   4 +-
 tests/test_downloadermiddleware_offsite.py | 178 +++++++++++++++++++++
 2 files changed, 181 insertions(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index 335007d8d..0bf25fb8f 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -124,7 +124,9 @@ class OffsiteMiddleware:
             allowed_domains = process_domains(allowed_domains_arg, "allowed_domains")
 
         if disallowed_domains_arg:
-            disallowed_domains = process_domains(disallowed_domains_arg, "disallowed_domains")
+            disallowed_domains = process_domains(
+                disallowed_domains_arg, "disallowed_domains"
+            )
 
         if allowed_domains:
             allowed_domain_pattern = rf"^(.*\.)?({'|'.join(allowed_domains)})$"
diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index dc0a31a76..b21979196 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -219,3 +219,181 @@ def test_request_scheduled_invalid_domains():
         request = Request(f"https://{letter}.example")
         with pytest.raises(IgnoreRequest):
             mw.request_scheduled(request, crawler.spider)
+
+
+@pytest.mark.parametrize(
+    ("disallowed_domain", "url", "allowed"),
+    [
+        ("example.com", "http://example.com/1", False),
+        ("example.com", "http://example.org/1", True),
+        ("example.com", "http://sub.example.com/1", False),
+        ("sub.example.com", "http://sub.example.com/1", False),
+        ("sub.example.com", "http://example.com/1", True),
+        ("example.com", "http://example.com:8000/1", False),
+        ("example.com", "http://example.org/example.com", True),
+        ("example.com", "http://example.org/foo.example.com", True),
+        ("example.com", "http://example.com.example", True),
+        ("a.example", "http://nota.example", True),
+        ("b.a.example", "http://notb.a.example", True),
+    ],
+)
+def test_process_request_disallowed_domains(disallowed_domain, url, allowed):
+    crawler = get_crawler(Spider)
+    crawler.spider = crawler._create_spider(
+        name="a", disallowed_domains=[disallowed_domain]
+    )
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(crawler.spider)
+    request = Request(url)
+    if allowed:
+        assert mw.process_request(request) is None
+    else:
+        with pytest.raises(IgnoreRequest):
+            mw.process_request(request)
+
+
+@pytest.mark.parametrize(
+    ("value", "filtered"),
+    [
+        (UNSET, True),
+        (None, True),
+        (False, True),
+        (True, False),
+    ],
+)
+def test_process_request_disallowed_dont_filter(value, filtered):
+    crawler = get_crawler(Spider)
+    crawler.spider = crawler._create_spider(name="a", disallowed_domains=["a.example"])
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(crawler.spider)
+
+    kwargs = {}
+    if value is not UNSET:
+        kwargs["dont_filter"] = value
+    request = Request("https://a.example", **kwargs)
+
+    if filtered:
+        with pytest.raises(IgnoreRequest):
+            mw.process_request(request)
+    else:
+        assert mw.process_request(request) is None
+
+
+@pytest.mark.parametrize(
+    ("allow_offsite", "dont_filter", "filtered"),
+    [
+        (True, UNSET, False),
+        (True, None, False),
+        (True, False, False),
+        (True, True, False),
+        (False, UNSET, True),
+        (False, None, True),
+        (False, False, True),
+        (False, True, False),
+    ],
+)
+def test_process_request_disallowed_allow_offsite(allow_offsite, dont_filter, filtered):
+    crawler = get_crawler(Spider)
+    crawler.spider = crawler._create_spider(name="a", disallowed_domains=["a.example"])
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(crawler.spider)
+
+    kwargs = {"meta": {}}
+    if allow_offsite is not UNSET:
+        kwargs["meta"]["allow_offsite"] = allow_offsite
+    if dont_filter is not UNSET:
+        kwargs["dont_filter"] = dont_filter
+    request = Request("https://a.example", **kwargs)
+
+    if filtered:
+        with pytest.raises(IgnoreRequest):
+            mw.process_request(request)
+    else:
+        assert mw.process_request(request) is None
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        UNSET,
+        None,
+        [],
+    ],
+)
+def test_process_request_no_disallowed_domains(value):
+    crawler = get_crawler(Spider)
+    kwargs = {}
+    if value is not UNSET:
+        kwargs["disallowed_domains"] = value
+    crawler.spider = crawler._create_spider(name="a", **kwargs)
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(crawler.spider)
+    request = Request("https://example.com")
+    assert mw.process_request(request) is None
+
+
+def test_process_request_invalid_disallowed_domains():
+    crawler = get_crawler(Spider)
+    disallowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
+    crawler.spider = crawler._create_spider(
+        name="a", disallowed_domains=disallowed_domains
+    )
+
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore", UserWarning)
+        mw.spider_opened(crawler.spider)
+
+    request = Request("https://a.example")
+    with pytest.raises(IgnoreRequest):
+        mw.process_request(request)
+
+    for letter in ("b", "c"):
+        request = Request(f"https://{letter}.example")
+        with pytest.raises(IgnoreRequest):
+            mw.process_request(request)
+
+
+@pytest.mark.parametrize(
+    ("disallowed_domain", "url", "allowed"),
+    [
+        ("example.com", "http://example.com/1", False),
+        ("example.com", "http://example.org/1", True),
+        ("example.com", "http://sub.example.com/1", False),
+        ("sub.example.com", "http://sub.example.com/1", False),
+        ("sub.example.com", "http://example.com/1", True),
+    ],
+)
+def test_request_scheduled_disallowed_domains(disallowed_domain, url, allowed):
+    crawler = get_crawler(Spider)
+    crawler.spider = crawler._create_spider(
+        name="a", disallowed_domains=[disallowed_domain]
+    )
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(crawler.spider)
+    request = Request(url)
+    if allowed:
+        assert mw.request_scheduled(request, crawler.spider) is None
+    else:
+        with pytest.raises(IgnoreRequest):
+            mw.request_scheduled(request, crawler.spider)
+
+
+@pytest.mark.parametrize(
+    "value",
+    [
+        UNSET,
+        None,
+        [],
+    ],
+)
+def test_request_scheduled_no_disallowed_domains(value):
+    crawler = get_crawler(Spider)
+    kwargs = {}
+    if value is not UNSET:
+        kwargs["disallowed_domains"] = value
+    crawler.spider = crawler._create_spider(name="a", **kwargs)
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(crawler.spider)
+    request = Request("https://example.com")
+    assert mw.request_scheduled(request, crawler.spider) is None

From 784dbe54b0cc85ea13d82d45e07449d8d79ac589 Mon Sep 17 00:00:00 2001
From: Diogo Castro <diogo.castro@zyte.com>
Date: Wed, 12 Nov 2025 11:19:06 -0300
Subject: [PATCH 03/13] feat: Removing redundant tests

---
 tests/test_downloadermiddleware_offsite.py | 56 ----------------------
 1 file changed, 56 deletions(-)

diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index b21979196..096f1bfb3 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -221,37 +221,6 @@ def test_request_scheduled_invalid_domains():
             mw.request_scheduled(request, crawler.spider)
 
 
-@pytest.mark.parametrize(
-    ("disallowed_domain", "url", "allowed"),
-    [
-        ("example.com", "http://example.com/1", False),
-        ("example.com", "http://example.org/1", True),
-        ("example.com", "http://sub.example.com/1", False),
-        ("sub.example.com", "http://sub.example.com/1", False),
-        ("sub.example.com", "http://example.com/1", True),
-        ("example.com", "http://example.com:8000/1", False),
-        ("example.com", "http://example.org/example.com", True),
-        ("example.com", "http://example.org/foo.example.com", True),
-        ("example.com", "http://example.com.example", True),
-        ("a.example", "http://nota.example", True),
-        ("b.a.example", "http://notb.a.example", True),
-    ],
-)
-def test_process_request_disallowed_domains(disallowed_domain, url, allowed):
-    crawler = get_crawler(Spider)
-    crawler.spider = crawler._create_spider(
-        name="a", disallowed_domains=[disallowed_domain]
-    )
-    mw = OffsiteMiddleware.from_crawler(crawler)
-    mw.spider_opened(crawler.spider)
-    request = Request(url)
-    if allowed:
-        assert mw.process_request(request) is None
-    else:
-        with pytest.raises(IgnoreRequest):
-            mw.process_request(request)
-
-
 @pytest.mark.parametrize(
     ("value", "filtered"),
     [
@@ -354,31 +323,6 @@ def test_process_request_invalid_disallowed_domains():
             mw.process_request(request)
 
 
-@pytest.mark.parametrize(
-    ("disallowed_domain", "url", "allowed"),
-    [
-        ("example.com", "http://example.com/1", False),
-        ("example.com", "http://example.org/1", True),
-        ("example.com", "http://sub.example.com/1", False),
-        ("sub.example.com", "http://sub.example.com/1", False),
-        ("sub.example.com", "http://example.com/1", True),
-    ],
-)
-def test_request_scheduled_disallowed_domains(disallowed_domain, url, allowed):
-    crawler = get_crawler(Spider)
-    crawler.spider = crawler._create_spider(
-        name="a", disallowed_domains=[disallowed_domain]
-    )
-    mw = OffsiteMiddleware.from_crawler(crawler)
-    mw.spider_opened(crawler.spider)
-    request = Request(url)
-    if allowed:
-        assert mw.request_scheduled(request, crawler.spider) is None
-    else:
-        with pytest.raises(IgnoreRequest):
-            mw.request_scheduled(request, crawler.spider)
-
-
 @pytest.mark.parametrize(
     "value",
     [

From 0995e96a4304446c742992a884cec2bfff14c500 Mon Sep 17 00:00:00 2001
From: Diogo Castro <diogo.castro@zyte.com>
Date: Wed, 19 Nov 2025 11:04:57 -0300
Subject: [PATCH 04/13] feat: Adding better examples to get_host_regex
 docstrings

---
 scrapy/downloadermiddlewares/offsite.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index 0bf25fb8f..c94335188 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -80,7 +80,7 @@ class OffsiteMiddleware:
         disallowed_domains = ['example2.com']
         This will allow crawling all subdomains of example.com (eg. foo.example.com,
         bar.example.com). But it won't allow crawling example2.com or any subdomain
-        (eg. www.example2.com).
+        (eg. foo.example2.com, bar.example2.com).
         """
         allowed_domains_arg = getattr(spider, "allowed_domains", None)
         disallowed_domains_arg = getattr(spider, "disallowed_domains", None)

From cdfc1efd19a7477e4c57fbbd7138d9e29d27b884 Mon Sep 17 00:00:00 2001
From: Adrian Chaves <adrian@zyte.com>
Date: Tue, 20 Jan 2026 14:43:08 +0100
Subject: [PATCH 05/13] Test that disallowing 1 domain does not disallow all
 domains

---
 tests/test_downloadermiddleware_offsite.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index 096f1bfb3..da850da87 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -247,6 +247,9 @@ def test_process_request_disallowed_dont_filter(value, filtered):
     else:
         assert mw.process_request(request) is None
 
+    request2 = Request("https://b.example")
+    assert mw.process_request(request2) is None
+
 
 @pytest.mark.parametrize(
     ("allow_offsite", "dont_filter", "filtered"),

From 4139fcb3a1408d73514830f814f7fa06a1b88edf Mon Sep 17 00:00:00 2001
From: Diogo Castro <diogo.neves.castro@gmail.com>
Date: Thu, 12 Mar 2026 11:23:57 -0300
Subject: [PATCH 06/13] Handle disallowed_domains separately from
 allowed_domains regex

---
 scrapy/downloadermiddlewares/offsite.py    | 127 ++++++++++-----------
 tests/test_downloadermiddleware_offsite.py |   3 +-
 2 files changed, 61 insertions(+), 69 deletions(-)

diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index c94335188..315838a8d 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -39,6 +39,9 @@ class OffsiteMiddleware:
 
     def spider_opened(self, spider: Spider) -> None:
         self.host_regex: re.Pattern[str] = self.get_host_regex(spider)
+        self.disallowed_host_regex: re.Pattern[str] | None = (
+            self._get_disallowed_host_regex(spider)
+        )
 
     def request_scheduled(self, request: Request, spider: Spider) -> None:
         self.process_request(request)
@@ -65,81 +68,71 @@ class OffsiteMiddleware:
         raise IgnoreRequest
 
     def should_follow(self, request: Request, spider: Spider) -> bool:
-        regex = self.host_regex
         # hostname can be None for wrong urls (like javascript links)
         host = urlparse_cached(request).hostname or ""
-        return bool(regex.search(host))
+        # If the host matches a disallowed domain, we can reject it
+        if self.disallowed_host_regex and self.disallowed_host_regex.search(host):
+            return False
+        # Otherwise, check allowed domains
+        return bool(self.host_regex.search(host))
+
+    @staticmethod
+    def _process_domains(
+        domains_list: list[str | None], domains_type: str
+    ) -> list[str]:
+        """Process a domains list and return a list of valid, regex-escaped domains.
+
+        Entries that are URLs or contain ports are warned about and skipped.
+        """
+        url_pattern = re.compile(r"^https?://.*$")
+        port_pattern = re.compile(r":\d+$")
+        valid_domains: list[str] = []
+
+        for domain in domains_list:
+            if domain is None:
+                continue
+            if url_pattern.match(domain):
+                message = (
+                    f"{domains_type} accepts only domains, not URLs. "
+                    f"Ignoring URL entry {domain} in {domains_type}."
+                )
+                warnings.warn(message)
+            elif port_pattern.search(domain):
+                message = (
+                    f"{domains_type} accepts only domains without ports. "
+                    f"Ignoring entry {domain} in {domains_type}."
+                )
+                warnings.warn(message)
+            else:
+                valid_domains.append(re.escape(domain))
+        return valid_domains
 
     def get_host_regex(self, spider: Spider) -> re.Pattern[str]:
         """Override this method to implement a different offsite policy.
-        Returns a compiled regular expression object that matches the hosts that
-        are allowed to be crawled. If None is returned (or method is not overridden),
-        all hosts are allowed.
-        Example:
-        allowed_domains = ['example.com']
-        disallowed_domains = ['example2.com']
-        This will allow crawling all subdomains of example.com (eg. foo.example.com,
-        bar.example.com). But it won't allow crawling example2.com or any subdomain
-        (eg. foo.example2.com, bar.example2.com).
+
+        Returns a compiled regular expression object that matches the hosts
+        that are allowed to be crawled.
         """
-        allowed_domains_arg = getattr(spider, "allowed_domains", None)
-        disallowed_domains_arg = getattr(spider, "disallowed_domains", None)
-        allowed_domains = []
-        disallowed_domains = []
-        if not allowed_domains_arg and not disallowed_domains_arg:
+        allowed_domains = getattr(spider, "allowed_domains", None)
+        if not allowed_domains:
             return re.compile("")  # allow all by default
 
-        url_pattern = re.compile(r"^https?://.*$")
-        port_pattern = re.compile(r":\d+$")
-        # domains = []
+        domains = self._process_domains(allowed_domains, "allowed_domains")
+        if domains:
+            return re.compile(rf"^(.*\.)?({'|'.join(domains)})$")
+        return re.compile("")  # allow all if no valid domains remain
 
-        def process_domains(domains_list=[], domains_type="allowed_domains"):
-            """
-            Process the domains list and return a list of valid domains.
-            The arguments passed to the spider in allowed_domains and disallowed_domains
-            cannot be URLs or contain ports.
-            """
-            valid_domains = []
+    def _get_disallowed_host_regex(self, spider: Spider) -> re.Pattern[str] | None:
+        """Build a regex that positively matches disallowed hosts.
 
-            for domain in domains_list:
-                if domain is None:
-                    continue
-                if url_pattern.match(domain):
-                    message = (
-                        f"{domains_type} accepts only domains, not URLs. "
-                        f"Ignoring URL entry {domain} in {domains_type}."
-                    )
-                    warnings.warn(message)
-                elif port_pattern.search(domain):
-                    message = (
-                        f"{domains_type} accepts only domains without ports. "
-                        f"Ignoring entry {domain} in {domains_type}."
-                    )
-                    warnings.warn(message)
-                else:
-                    valid_domains.append(re.escape(domain))
-            return valid_domains
+        Returns ``None`` when there are no disallowed domains, meaning
+        nothing should be blocked via this mechanism.
+        """
+        disallowed_domains = getattr(spider, "disallowed_domains", None)
+        if not disallowed_domains:
+            return None
 
-        if allowed_domains_arg:
-            allowed_domains = process_domains(allowed_domains_arg, "allowed_domains")
-
-        if disallowed_domains_arg:
-            disallowed_domains = process_domains(
-                disallowed_domains_arg, "disallowed_domains"
-            )
-
-        if allowed_domains:
-            allowed_domain_pattern = rf"^(.*\.)?({'|'.join(allowed_domains)})$"
-        else:
-            allowed_domain_pattern = ""
-
-        if disallowed_domains:
-            disallowed_domain_pattern = rf"^(?!.*(?:{'|'.join(disallowed_domains)}))$"
-        else:
-            disallowed_domain_pattern = ""
-
-        if allowed_domain_pattern and disallowed_domain_pattern:
-            combined_pattern = rf"{allowed_domain_pattern}|{disallowed_domain_pattern}"
-        else:
-            combined_pattern = allowed_domain_pattern or disallowed_domain_pattern
-        return re.compile(combined_pattern)
+        domains = self._process_domains(disallowed_domains, "disallowed_domains")
+        if domains:
+            return re.compile(rf"^(.*\.)?({'|'.join(domains)})$")
+        return None
diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index da850da87..b8e2e0480 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -322,8 +322,7 @@ def test_process_request_invalid_disallowed_domains():
 
     for letter in ("b", "c"):
         request = Request(f"https://{letter}.example")
-        with pytest.raises(IgnoreRequest):
-            mw.process_request(request)
+        assert mw.process_request(request) is None
 
 
 @pytest.mark.parametrize(

From 8bb1737113605541911c8ac752e252b17c604d73 Mon Sep 17 00:00:00 2001
From: Diogo Castro <diogo.neves.castro@gmail.com>
Date: Wed, 15 Apr 2026 19:36:28 -0300
Subject: [PATCH 07/13] fix: Start raising ValueError on invalid domain entries
 and close spider

---
 scrapy/downloadermiddlewares/offsite.py    | 43 ++++++++++++-------
 tests/test_downloadermiddleware_offsite.py | 49 ++++++++--------------
 2 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index 315838a8d..dca92631f 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -2,12 +2,12 @@ from __future__ import annotations
 
 import logging
 import re
-import warnings
 from typing import TYPE_CHECKING
 
 from scrapy import Request, Spider, signals
 from scrapy.exceptions import IgnoreRequest
 from scrapy.utils.decorators import _warn_spider_arg
+from scrapy.utils.defer import _schedule_coro
 from scrapy.utils.httpobj import urlparse_cached
 
 if TYPE_CHECKING:
@@ -38,10 +38,23 @@ class OffsiteMiddleware:
         return o
 
     def spider_opened(self, spider: Spider) -> None:
-        self.host_regex: re.Pattern[str] = self.get_host_regex(spider)
-        self.disallowed_host_regex: re.Pattern[str] | None = (
-            self._get_disallowed_host_regex(spider)
-        )
+        try:
+            self.host_regex: re.Pattern[str] = self.get_host_regex(spider)
+            self.disallowed_host_regex: re.Pattern[str] | None = (
+                self._get_disallowed_host_regex(spider)
+            )
+        except ValueError as exc:
+            logger.error(
+                "Invalid domain configuration: %(error)s",
+                {"error": exc},
+                extra={"spider": spider},
+            )
+            if self.crawler.engine:
+                _schedule_coro(
+                    self.crawler.engine.close_spider_async(
+                        reason="invalid_domain_configuration"
+                    )
+                )
 
     def request_scheduled(self, request: Request, spider: Spider) -> None:
         self.process_request(request)
@@ -78,11 +91,14 @@ class OffsiteMiddleware:
 
     @staticmethod
     def _process_domains(
-        domains_list: list[str | None], domains_type: str
+        domains_list: list[str | None],
+        domains_type: str,
     ) -> list[str]:
         """Process a domains list and return a list of valid, regex-escaped domains.
 
-        Entries that are URLs or contain ports are warned about and skipped.
+        Raises ``ValueError`` on invalid entries (``None``, URLs, or
+        domains with ports) so that the spider fails fast on
+        misconfigured domains.
         """
         url_pattern = re.compile(r"^https?://.*$")
         port_pattern = re.compile(r":\d+$")
@@ -90,21 +106,18 @@ class OffsiteMiddleware:
 
         for domain in domains_list:
             if domain is None:
-                continue
+                raise ValueError(f"{domains_type} contains empty value.")
             if url_pattern.match(domain):
-                message = (
+                raise ValueError(
                     f"{domains_type} accepts only domains, not URLs. "
                     f"Ignoring URL entry {domain} in {domains_type}."
                 )
-                warnings.warn(message)
-            elif port_pattern.search(domain):
-                message = (
+            if port_pattern.search(domain):
+                raise ValueError(
                     f"{domains_type} accepts only domains without ports. "
                     f"Ignoring entry {domain} in {domains_type}."
                 )
-                warnings.warn(message)
-            else:
-                valid_domains.append(re.escape(domain))
+            valid_domains.append(re.escape(domain))
         return valid_domains
 
     def get_host_regex(self, spider: Spider) -> re.Pattern[str]:
diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index b8e2e0480..eaa34d125 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -1,4 +1,4 @@
-import warnings
+import logging
 
 import pytest
 
@@ -115,20 +115,14 @@ def test_process_request_no_allowed_domains(value):
     assert mw.process_request(request) is None
 
 
-def test_process_request_invalid_domains():
+def test_process_request_invalid_domains(caplog):
     crawler = get_crawler(Spider)
     allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
     crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
     mw = OffsiteMiddleware.from_crawler(crawler)
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore", UserWarning)
+    with caplog.at_level(logging.ERROR):
         mw.spider_opened(crawler.spider)
-    request = Request("https://a.example")
-    assert mw.process_request(request) is None
-    for letter in ("b", "c"):
-        request = Request(f"https://{letter}.example")
-        with pytest.raises(IgnoreRequest):
-            mw.process_request(request)
+    assert "Invalid domain configuration" in caplog.text
 
 
 @pytest.mark.parametrize(
@@ -205,20 +199,14 @@ def test_request_scheduled_no_allowed_domains(value):
     assert mw.request_scheduled(request, crawler.spider) is None
 
 
-def test_request_scheduled_invalid_domains():
+def test_request_scheduled_invalid_domains(caplog):
     crawler = get_crawler(Spider)
     allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
     crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
     mw = OffsiteMiddleware.from_crawler(crawler)
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore", UserWarning)
+    with caplog.at_level(logging.ERROR):
         mw.spider_opened(crawler.spider)
-    request = Request("https://a.example")
-    assert mw.request_scheduled(request, crawler.spider) is None
-    for letter in ("b", "c"):
-        request = Request(f"https://{letter}.example")
-        with pytest.raises(IgnoreRequest):
-            mw.request_scheduled(request, crawler.spider)
+    assert "Invalid domain configuration" in caplog.text
 
 
 @pytest.mark.parametrize(
@@ -304,25 +292,24 @@ def test_process_request_no_disallowed_domains(value):
     assert mw.process_request(request) is None
 
 
-def test_process_request_invalid_disallowed_domains():
+@pytest.mark.parametrize(
+    "disallowed_domains",
+    [
+        ["a.example", None],
+        ["a.example", "http:////b.example"],
+        ["a.example", "//c.example:8080"],
+    ],
+)
+def test_process_request_invalid_disallowed_domains(disallowed_domains, caplog):
     crawler = get_crawler(Spider)
-    disallowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
     crawler.spider = crawler._create_spider(
         name="a", disallowed_domains=disallowed_domains
     )
 
     mw = OffsiteMiddleware.from_crawler(crawler)
-    with warnings.catch_warnings():
-        warnings.simplefilter("ignore", UserWarning)
+    with caplog.at_level(logging.ERROR):
         mw.spider_opened(crawler.spider)
-
-    request = Request("https://a.example")
-    with pytest.raises(IgnoreRequest):
-        mw.process_request(request)
-
-    for letter in ("b", "c"):
-        request = Request(f"https://{letter}.example")
-        assert mw.process_request(request) is None
+    assert "Invalid domain configuration" in caplog.text
 
 
 @pytest.mark.parametrize(

From 17c9704a1320bf3bfa511b867e78d844a2c64c7c Mon Sep 17 00:00:00 2001
From: Diogo Castro <diogo.neves.castro@gmail.com>
Date: Fri, 17 Apr 2026 09:59:44 -0300
Subject: [PATCH 08/13] feat: Adding missing test counterparts for
 disallowed_domains based on the ones that exists for allowed_domains

---
 tests/test_downloadermiddleware_offsite.py | 85 ++++++++++++++++++++++
 1 file changed, 85 insertions(+)

diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index eaa34d125..cfa8f146a 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -330,3 +330,88 @@ def test_request_scheduled_no_disallowed_domains(value):
     mw.spider_opened(crawler.spider)
     request = Request("https://example.com")
     assert mw.request_scheduled(request, crawler.spider) is None
+
+
+@pytest.mark.parametrize(
+    ("value", "filtered"),
+    [
+        (UNSET, True),
+        (None, True),
+        (False, True),
+        (True, False),
+    ],
+)
+def test_request_scheduled_disallowed_dont_filter(value, filtered):
+    crawler = get_crawler(Spider)
+    crawler.spider = crawler._create_spider(name="a", disallowed_domains=["a.example"])
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(crawler.spider)
+
+    kwargs = {}
+    if value is not UNSET:
+        kwargs["dont_filter"] = value
+    request = Request("https://a.example", **kwargs)
+
+    if filtered:
+        with pytest.raises(IgnoreRequest):
+            mw.request_scheduled(request, crawler.spider)
+    else:
+        assert mw.request_scheduled(request, crawler.spider) is None
+
+    request2 = Request("https://b.example")
+    assert mw.request_scheduled(request2, crawler.spider) is None
+
+
+@pytest.mark.parametrize(
+    ("allow_offsite", "dont_filter", "filtered"),
+    [
+        (True, UNSET, False),
+        (True, None, False),
+        (True, False, False),
+        (True, True, False),
+        (False, UNSET, True),
+        (False, None, True),
+        (False, False, True),
+        (False, True, False),
+    ],
+)
+def test_request_scheduled_disallowed_allow_offsite(
+    allow_offsite, dont_filter, filtered
+):
+    crawler = get_crawler(Spider)
+    crawler.spider = crawler._create_spider(name="a", disallowed_domains=["a.example"])
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(crawler.spider)
+
+    kwargs = {"meta": {}}
+    if allow_offsite is not UNSET:
+        kwargs["meta"]["allow_offsite"] = allow_offsite
+    if dont_filter is not UNSET:
+        kwargs["dont_filter"] = dont_filter
+    request = Request("https://a.example", **kwargs)
+
+    if filtered:
+        with pytest.raises(IgnoreRequest):
+            mw.request_scheduled(request, crawler.spider)
+    else:
+        assert mw.request_scheduled(request, crawler.spider) is None
+
+
+@pytest.mark.parametrize(
+    "disallowed_domains",
+    [
+        ["a.example", None],
+        ["a.example", "http:////b.example"],
+        ["a.example", "//c.example:8080"],
+    ],
+)
+def test_request_scheduled_invalid_disallowed_domains(disallowed_domains, caplog):
+    crawler = get_crawler(Spider)
+    crawler.spider = crawler._create_spider(
+        name="a", disallowed_domains=disallowed_domains
+    )
+
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    with caplog.at_level(logging.ERROR):
+        mw.spider_opened(crawler.spider)
+    assert "Invalid domain configuration" in caplog.text

From b3a9a6de869737836b34c8b33f95e4819d21f0d5 Mon Sep 17 00:00:00 2001
From: Diogo Castro <diogo.neves.castro@gmail.com>
Date: Fri, 19 Jun 2026 00:32:29 -0300
Subject: [PATCH 09/13] Fix codecov: remove dead code, add test for engine
 close branch

---
 scrapy/downloadermiddlewares/offsite.py    |  8 ++------
 tests/test_downloadermiddleware_offsite.py | 20 ++++++++++++++++++++
 2 files changed, 22 insertions(+), 6 deletions(-)

diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index dca92631f..ea2c2f030 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -131,9 +131,7 @@ class OffsiteMiddleware:
             return re.compile("")  # allow all by default
 
         domains = self._process_domains(allowed_domains, "allowed_domains")
-        if domains:
-            return re.compile(rf"^(.*\.)?({'|'.join(domains)})$")
-        return re.compile("")  # allow all if no valid domains remain
+        return re.compile(rf"^(.*\.)?({'|'.join(domains)})$")
 
     def _get_disallowed_host_regex(self, spider: Spider) -> re.Pattern[str] | None:
         """Build a regex that positively matches disallowed hosts.
@@ -146,6 +144,4 @@ class OffsiteMiddleware:
             return None
 
         domains = self._process_domains(disallowed_domains, "disallowed_domains")
-        if domains:
-            return re.compile(rf"^(.*\.)?({'|'.join(domains)})$")
-        return None
+        return re.compile(rf"^(.*\.)?({'|'.join(domains)})$")
diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index cfa8f146a..1d459c8cb 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -1,4 +1,5 @@
 import logging
+from unittest.mock import AsyncMock, patch
 
 import pytest
 
@@ -125,6 +126,25 @@ def test_process_request_invalid_domains(caplog):
     assert "Invalid domain configuration" in caplog.text
 
 
+def test_invalid_domains_closes_spider(caplog):
+    crawler = get_crawler(Spider)
+    crawler.spider = crawler._create_spider(
+        name="a", allowed_domains=["a.example", None]
+    )
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mock_engine = AsyncMock()
+    crawler.engine = mock_engine
+    with (
+        patch(
+            "scrapy.downloadermiddlewares.offsite._schedule_coro"
+        ) as mock_schedule,
+        caplog.at_level(logging.ERROR),
+    ):
+        mw.spider_opened(crawler.spider)
+    assert "Invalid domain configuration" in caplog.text
+    mock_schedule.assert_called_once()
+
+
 @pytest.mark.parametrize(
     ("allowed_domain", "url", "allowed"),
     [

From fc32e89ac92edcf095a7ea45627c36407eb48f4a Mon Sep 17 00:00:00 2001
From: Diogo Castro <diogo.neves.castro@gmail.com>
Date: Fri, 19 Jun 2026 00:45:35 -0300
Subject: [PATCH 10/13] feat: Adding documentation to  feature

---
 docs/topics/downloader-middleware.rst      | 15 +++++++++++++++
 docs/topics/spiders.rst                    | 12 ++++++++++++
 tests/test_downloadermiddleware_offsite.py |  4 +---
 3 files changed, 28 insertions(+), 3 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 0c1af5276..f6fafbb52 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -856,6 +856,21 @@ OffsiteMiddleware
    :attr:`~scrapy.Spider.allowed_domains` attribute, or the
    attribute is empty, the offsite middleware will allow all requests.
 
+   If the spider defines a :attr:`~scrapy.Spider.disallowed_domains`
+   attribute, any request whose host name matches one of the domains in
+   that list (or their subdomains) will be filtered out, regardless of
+   :attr:`~scrapy.Spider.allowed_domains`. This is useful when you want
+   to allow most domains but block a few specific ones.
+
+   If a request matches both :attr:`~scrapy.Spider.allowed_domains` and
+   :attr:`~scrapy.Spider.disallowed_domains`, it will be filtered out
+   (i.e. :attr:`~scrapy.Spider.disallowed_domains` takes precedence).
+
+   Both :attr:`~scrapy.Spider.allowed_domains` and
+   :attr:`~scrapy.Spider.disallowed_domains` must contain valid domain
+   names only (not URLs or domains with ports). Invalid entries will cause
+   the spider to close with reason ``invalid_domain_configuration``.
+
    .. reqmeta:: allow_offsite
 
    If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 506daf930..56f41590a 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -74,6 +74,18 @@ scrapy.Spider
        Let's say your target url is ``https://www.example.com/1.html``,
        then add ``'example.com'`` to the list.
 
+   .. attribute:: disallowed_domains
+
+       An optional list of strings containing domains that this spider is
+       not allowed to crawl. Requests for URLs belonging to the domain names
+       specified in this list (or their subdomains) will be filtered out when
+       :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is
+       enabled.
+
+       This is useful when you want to allow all domains except a few
+       specific ones. For example, to block requests to ``ads.example.com``,
+       add ``'ads.example.com'`` to the list.
+
    .. autoattribute:: start_urls
 
    .. attribute:: custom_settings
diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index 1d459c8cb..0d7e8982b 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -135,9 +135,7 @@ def test_invalid_domains_closes_spider(caplog):
     mock_engine = AsyncMock()
     crawler.engine = mock_engine
     with (
-        patch(
-            "scrapy.downloadermiddlewares.offsite._schedule_coro"
-        ) as mock_schedule,
+        patch("scrapy.downloadermiddlewares.offsite._schedule_coro") as mock_schedule,
         caplog.at_level(logging.ERROR),
     ):
         mw.spider_opened(crawler.spider)

From 0a152764fb5aac7fc5cf575c63155a3374caca7b Mon Sep 17 00:00:00 2001
From: Diogo Castro <diogo.neves.castro@gmail.com>
Date: Sun, 21 Jun 2026 23:01:17 -0300
Subject: [PATCH 11/13] fix: Addressing feedbacks, changing error message, and
 adding tests

---
 scrapy/downloadermiddlewares/offsite.py    | 14 ++---
 tests/test_downloadermiddleware_offsite.py | 73 +++++++++++++++++++---
 2 files changed, 72 insertions(+), 15 deletions(-)

diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index ea2c2f030..fc9f35a66 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -49,12 +49,12 @@ class OffsiteMiddleware:
                 {"error": exc},
                 extra={"spider": spider},
             )
-            if self.crawler.engine:
-                _schedule_coro(
-                    self.crawler.engine.close_spider_async(
-                        reason="invalid_domain_configuration"
-                    )
+            assert self.crawler.engine
+            _schedule_coro(
+                self.crawler.engine.close_spider_async(
+                    reason="invalid_domain_configuration"
                 )
+            )
 
     def request_scheduled(self, request: Request, spider: Spider) -> None:
         self.process_request(request)
@@ -110,12 +110,12 @@ class OffsiteMiddleware:
             if url_pattern.match(domain):
                 raise ValueError(
                     f"{domains_type} accepts only domains, not URLs. "
-                    f"Ignoring URL entry {domain} in {domains_type}."
+                    f"Got URL entry {domain} in {domains_type}."
                 )
             if port_pattern.search(domain):
                 raise ValueError(
                     f"{domains_type} accepts only domains without ports. "
-                    f"Ignoring entry {domain} in {domains_type}."
+                    f"Got entry {domain} in {domains_type}."
                 )
             valid_domains.append(re.escape(domain))
         return valid_domains
diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index 0d7e8982b..20172c02c 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -116,12 +116,23 @@ def test_process_request_no_allowed_domains(value):
     assert mw.process_request(request) is None
 
 
-def test_process_request_invalid_domains(caplog):
+@pytest.mark.parametrize(
+    "allowed_domains",
+    [
+        ["a.example", None],
+        ["a.example", "http://b.example"],
+        ["a.example", "c.example:8080"],
+    ],
+)
+def test_process_request_invalid_domains(allowed_domains, caplog):
     crawler = get_crawler(Spider)
-    allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
     crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
     mw = OffsiteMiddleware.from_crawler(crawler)
-    with caplog.at_level(logging.ERROR):
+    crawler.engine = AsyncMock()
+    with (
+        patch("scrapy.downloadermiddlewares.offsite._schedule_coro"),
+        caplog.at_level(logging.ERROR),
+    ):
         mw.spider_opened(crawler.spider)
     assert "Invalid domain configuration" in caplog.text
 
@@ -217,12 +228,23 @@ def test_request_scheduled_no_allowed_domains(value):
     assert mw.request_scheduled(request, crawler.spider) is None
 
 
-def test_request_scheduled_invalid_domains(caplog):
+@pytest.mark.parametrize(
+    "allowed_domains",
+    [
+        ["a.example", None],
+        ["a.example", "http://b.example"],
+        ["a.example", "c.example:8080"],
+    ],
+)
+def test_request_scheduled_invalid_domains(allowed_domains, caplog):
     crawler = get_crawler(Spider)
-    allowed_domains = ["a.example", None, "http:////b.example", "//c.example"]
     crawler.spider = crawler._create_spider(name="a", allowed_domains=allowed_domains)
     mw = OffsiteMiddleware.from_crawler(crawler)
-    with caplog.at_level(logging.ERROR):
+    crawler.engine = AsyncMock()
+    with (
+        patch("scrapy.downloadermiddlewares.offsite._schedule_coro"),
+        caplog.at_level(logging.ERROR),
+    ):
         mw.spider_opened(crawler.spider)
     assert "Invalid domain configuration" in caplog.text
 
@@ -325,7 +347,11 @@ def test_process_request_invalid_disallowed_domains(disallowed_domains, caplog):
     )
 
     mw = OffsiteMiddleware.from_crawler(crawler)
-    with caplog.at_level(logging.ERROR):
+    crawler.engine = AsyncMock()
+    with (
+        patch("scrapy.downloadermiddlewares.offsite._schedule_coro"),
+        caplog.at_level(logging.ERROR),
+    ):
         mw.spider_opened(crawler.spider)
     assert "Invalid domain configuration" in caplog.text
 
@@ -430,6 +456,37 @@ def test_request_scheduled_invalid_disallowed_domains(disallowed_domains, caplog
     )
 
     mw = OffsiteMiddleware.from_crawler(crawler)
-    with caplog.at_level(logging.ERROR):
+    crawler.engine = AsyncMock()
+    with (
+        patch("scrapy.downloadermiddlewares.offsite._schedule_coro"),
+        caplog.at_level(logging.ERROR),
+    ):
         mw.spider_opened(crawler.spider)
     assert "Invalid domain configuration" in caplog.text
+
+
+@pytest.mark.parametrize(
+    ("url", "filtered"),
+    [
+        ("http://example.com/page", False),
+        ("http://sub.example.com/page", False),
+        ("http://ads.example.com/page", True),
+        ("http://sub.ads.example.com/page", True),
+        ("http://other.com/page", True),
+    ],
+)
+def test_process_request_allowed_and_disallowed_domains(url, filtered):
+    crawler = get_crawler(Spider)
+    crawler.spider = crawler._create_spider(
+        name="a",
+        allowed_domains=["example.com"],
+        disallowed_domains=["ads.example.com"],
+    )
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(crawler.spider)
+    request = Request(url)
+    if filtered:
+        with pytest.raises(IgnoreRequest):
+            mw.process_request(request)
+    else:
+        assert mw.process_request(request) is None

From f38c2afe912fd16f85bd9efaea9fcf6ad22a7f29 Mon Sep 17 00:00:00 2001
From: Diogo Castro <diogo.neves.castro@gmail.com>
Date: Mon, 22 Jun 2026 10:10:02 -0300
Subject: [PATCH 12/13] fix: Adding protocol_relative_pattern to reject another
 pattern for invalid URLs

---
 scrapy/downloadermiddlewares/offsite.py    | 3 ++-
 tests/test_downloadermiddleware_offsite.py | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index fc9f35a66..5dcd67bc7 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -101,13 +101,14 @@ class OffsiteMiddleware:
         misconfigured domains.
         """
         url_pattern = re.compile(r"^https?://.*$")
+        protocol_relative_pattern = re.compile(r"^//")
         port_pattern = re.compile(r":\d+$")
         valid_domains: list[str] = []
 
         for domain in domains_list:
             if domain is None:
                 raise ValueError(f"{domains_type} contains empty value.")
-            if url_pattern.match(domain):
+            if url_pattern.match(domain) or protocol_relative_pattern.match(domain):
                 raise ValueError(
                     f"{domains_type} accepts only domains, not URLs. "
                     f"Got URL entry {domain} in {domains_type}."
diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index 20172c02c..62e236fdf 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -121,6 +121,7 @@ def test_process_request_no_allowed_domains(value):
     [
         ["a.example", None],
         ["a.example", "http://b.example"],
+        ["a.example", "//c.example"],
         ["a.example", "c.example:8080"],
     ],
 )
@@ -233,6 +234,7 @@ def test_request_scheduled_no_allowed_domains(value):
     [
         ["a.example", None],
         ["a.example", "http://b.example"],
+        ["a.example", "//c.example"],
         ["a.example", "c.example:8080"],
     ],
 )

From b8cedd8d271f04aaad45358b1898bd88daa1b2e8 Mon Sep 17 00:00:00 2001
From: Diogo Castro <diogo.neves.castro@gmail.com>
Date: Tue, 23 Jun 2026 10:31:14 -0300
Subject: [PATCH 13/13] feat: Removing unecessary comments

---
 scrapy/downloadermiddlewares/offsite.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index 5dcd67bc7..79f75be5a 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -83,10 +83,8 @@ class OffsiteMiddleware:
     def should_follow(self, request: Request, spider: Spider) -> bool:
         # hostname can be None for wrong urls (like javascript links)
         host = urlparse_cached(request).hostname or ""
-        # If the host matches a disallowed domain, we can reject it
         if self.disallowed_host_regex and self.disallowed_host_regex.search(host):
             return False
-        # Otherwise, check allowed domains
         return bool(self.host_regex.search(host))
 
     @staticmethod