diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 8fbf0c52d..f2cfeb712 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -59,9 +59,16 @@ scrapy.Spider :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is enabled. + .. versionchanged:: VERSION + Changes to this attribute during a crawl are now taken into account. + Let's say your target url is ``https://www.example.com/1.html``, then add ``'example.com'`` to the list. + You may modify this attribute while the spider runs, e.g. to allow + domains that you only learn about from an earlier response. The change + affects requests scheduled after it. + .. autoattribute:: start_urls .. attribute:: custom_settings diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index db85b62a1..28c0e09cb 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -22,10 +22,12 @@ logger = logging.getLogger(__name__) class OffsiteMiddleware: crawler: Crawler + host_regex: re.Pattern[str] def __init__(self, stats: StatsCollector): self.stats = stats self.domains_seen: set[str] = set() + self._allowed_domains: list[str] | None = None @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -37,7 +39,13 @@ class OffsiteMiddleware: return o def spider_opened(self, spider: Spider) -> None: - self.host_regex: re.Pattern[str] = self.get_host_regex(spider) + self._update_host_regex(spider) + + def _update_host_regex(self, spider: Spider) -> None: + allowed_domains = list(getattr(spider, "allowed_domains", None) or []) + if allowed_domains != self._allowed_domains: + self._allowed_domains = allowed_domains + self.host_regex = self.get_host_regex(spider) def request_scheduled(self, request: Request, spider: Spider) -> None: self.process_request(request) @@ -64,6 +72,7 @@ class OffsiteMiddleware: raise IgnoreRequest(f"Filtered offsite request to {domain!r}") def should_follow(self, request: Request, spider: Spider) -> bool: + self._update_host_regex(spider) regex = self.host_regex # hostname can be None for wrong urls (like javascript links) host = urlparse_cached(request).hostname or "" diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index cb17c2553..bab89814f 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -247,3 +247,50 @@ def test_ignore_request_reason(): IgnoreRequest, match=re.escape("Filtered offsite request to 'other.org'") ): mw.process_request(request) + + +class DomainSpider(Spider): + name = "a" + allowed_domains: list[str] + + +def test_dynamic_allowed_domains(): + crawler = get_crawler(DomainSpider) + spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"]) + crawler.spider = spider + mw = OffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + + with pytest.raises(IgnoreRequest): + mw.process_request(Request("https://b.example")) + + spider.allowed_domains.append("b.example") + assert mw.process_request(Request("https://b.example")) is None + + spider.allowed_domains.remove("a.example") + with pytest.raises(IgnoreRequest): + mw.process_request(Request("https://a.example")) + + +def test_dynamic_allowed_domains_caching(): + calls = 0 + + class TrackingMiddleware(OffsiteMiddleware): + def get_host_regex(self, spider: Spider) -> re.Pattern[str]: + nonlocal calls + calls += 1 + return super().get_host_regex(spider) + + crawler = get_crawler(DomainSpider) + spider = DomainSpider.from_crawler(crawler, allowed_domains=["a.example"]) + crawler.spider = spider + mw = TrackingMiddleware.from_crawler(crawler) + mw.spider_opened(spider) + + for _ in range(3): + mw.process_request(Request("https://a.example")) + assert calls == 1 + + spider.allowed_domains.append("b.example") + assert mw.process_request(Request("https://b.example")) is None + assert calls == 2