Let spiders change allowed_domains at run time (#7912)

docs/topics/spiders.rst

@@ -59,9 +59,16 @@ scrapy.Spider
        :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is
        enabled.
 
+       .. versionchanged:: VERSION
+          Changes to this attribute during a crawl are now taken into account.
+
        Let's say your target url is ``https://www.example.com/1.html``,
        then add ``'example.com'`` to the list.
 
+       You may modify this attribute while the spider runs, e.g. to allow
+       domains that you only learn about from an earlier response. The change
+       affects requests scheduled after it.
+
    .. autoattribute:: start_urls
 
    .. attribute:: custom_settings

scrapy/downloadermiddlewares/offsite.py

@@ -22,10 +22,12 @@ logger = logging.getLogger(__name__)
 
 class OffsiteMiddleware:
     crawler: Crawler
+    host_regex: re.Pattern[str]
 
     def __init__(self, stats: StatsCollector):
         self.stats = stats
         self.domains_seen: set[str] = set()
+        self._allowed_domains: list[str] | None = None
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
@@ -37,7 +39,13 @@ class OffsiteMiddleware:
         return o
 
     def spider_opened(self, spider: Spider) -> None:
-        self.host_regex: re.Pattern[str] = self.get_host_regex(spider)
+        self._update_host_regex(spider)
+
+    def _update_host_regex(self, spider: Spider) -> None:
+        allowed_domains = list(getattr(spider, "allowed_domains", None) or [])
+        if allowed_domains != self._allowed_domains:
+            self._allowed_domains = allowed_domains
+            self.host_regex = self.get_host_regex(spider)
 
     def request_scheduled(self, request: Request, spider: Spider) -> None:
         self.process_request(request)
@@ -64,6 +72,7 @@ class OffsiteMiddleware:
         raise IgnoreRequest(f"Filtered offsite request to {domain!r}")
 
     def should_follow(self, request: Request, spider: Spider) -> bool:
+        self._update_host_regex(spider)
         regex = self.host_regex
         # hostname can be None for wrong urls (like javascript links)
         host = urlparse_cached(request).hostname or ""

tests/test_downloadermiddleware_offsite.py

@@ -247,3 +247,50 @@ def test_ignore_request_reason():
         IgnoreRequest, match=re.escape("Filtered offsite request to 'other.org'")
     ):
         mw.process_request(request)
+
+
+class DomainSpider(Spider):
+    name = "a"
+    allowed_domains: list[str]
+
+
+def test_dynamic_allowed_domains():
+    crawler = get_crawler(DomainSpider)
+    spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"])
+    crawler.spider = spider
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(spider)
+
+    with pytest.raises(IgnoreRequest):
+        mw.process_request(Request("https://b.example"))
+
+    spider.allowed_domains.append("b.example")
+    assert mw.process_request(Request("https://b.example")) is None
+
+    spider.allowed_domains.remove("a.example")
+    with pytest.raises(IgnoreRequest):
+        mw.process_request(Request("https://a.example"))
+
+
+def test_dynamic_allowed_domains_caching():
+    calls = 0
+
+    class TrackingMiddleware(OffsiteMiddleware):
+        def get_host_regex(self, spider: Spider) -> re.Pattern[str]:
+            nonlocal calls
+            calls += 1
+            return super().get_host_regex(spider)
+
+    crawler = get_crawler(DomainSpider)
+    spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"])
+    crawler.spider = spider
+    mw = TrackingMiddleware.from_crawler(crawler)
+    mw.spider_opened(spider)
+
+    for _ in range(3):
+        mw.process_request(Request("https://a.example"))
+    assert calls == 1
+
+    spider.allowed_domains.append("b.example")
+    assert mw.process_request(Request("https://b.example")) is None
+    assert calls == 2