Document how to disallow subdomains of allowed domains (#7903)

docs/topics/downloader-middleware.rst

@@ -799,40 +799,9 @@ OffsiteMiddleware
 .. module:: scrapy.downloadermiddlewares.offsite
    :synopsis: Offsite Middleware
 
-.. class:: OffsiteMiddleware
+.. autoclass:: OffsiteMiddleware
 
-   .. versionadded:: 2.11.2
-
-   Filters out Requests for URLs outside the domains covered by the spider.
-
-   This middleware filters out every request whose host names aren't in the
-   spider's :attr:`~scrapy.Spider.allowed_domains` attribute.
-   All subdomains of any domain in the list are also allowed.
-   E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``
-   but not ``www2.example.com`` nor ``example.com``.
-
-   When your spider returns a request for a domain not belonging to those
-   covered by the spider, this middleware will log a debug message similar to
-   this one::
-
-      DEBUG: Filtered offsite request to 'offsite.example': <GET http://offsite.example/some/page.html>
-
-   To avoid filling the log with too much noise, it will only print one of
-   these messages for each new domain filtered. So, for example, if another
-   request for ``offsite.example`` is filtered, no log message will be
-   printed. But if a request for ``other.example`` is filtered, a message
-   will be printed (but only for the first request filtered).
-
-   If the spider doesn't define an
-   :attr:`~scrapy.Spider.allowed_domains` attribute, or the
-   attribute is empty, the offsite middleware will allow all requests.
-
-   .. reqmeta:: allow_offsite
-
-   If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to
-   ``True`` or :attr:`Request.meta <scrapy.Request.meta>` has ``allow_offsite``
-   set to ``True``, then the OffsiteMiddleware will allow the request even if
-   its domain is not listed in allowed domains.
+   .. automethod:: should_follow
 
 RedirectMiddleware
 ------------------

scrapy/downloadermiddlewares/offsite.py

@@ -21,6 +21,36 @@ logger = logging.getLogger(__name__)
 
 
 class OffsiteMiddleware:
+    """Filter out requests for URLs outside the domains covered by the spider.
+
+    .. versionadded:: 2.11.2
+
+    A request is allowed if its host name is in the
+    :attr:`~scrapy.Spider.allowed_domains` attribute of the spider, or is a
+    subdomain of one of those domains. E.g. ``www.example.org`` also allows
+    ``bob.www.example.org``, but neither ``www2.example.org`` nor
+    ``example.org``. See :meth:`should_follow` to use a different policy.
+
+    If the spider does not define :attr:`~scrapy.Spider.allowed_domains`, or
+    the attribute is empty, every request is allowed.
+
+    Filtered requests are logged as follows::
+
+        DEBUG: Filtered offsite request to 'offsite.example': <GET http://offsite.example/some/page.html>
+
+    Only the first request filtered for a given domain is logged, to keep the
+    log readable.
+
+    .. reqmeta:: allow_offsite
+
+    allow_offsite
+    -------------
+
+    Requests with the ``allow_offsite`` :attr:`~scrapy.Request.meta` key set to
+    ``True``, or with :attr:`~scrapy.Request.dont_filter` set to ``True``, are
+    allowed regardless of their host name.
+    """
+
     crawler: Crawler
     host_regex: re.Pattern[str]
 
@@ -72,6 +102,23 @@ class OffsiteMiddleware:
         raise IgnoreRequest(f"Filtered offsite request to {domain!r}")
 
     def should_follow(self, request: Request, spider: Spider) -> bool:
+        """Return ``True`` if *request* is on site, ``False`` if it must be
+        filtered out.
+
+        Override this method to implement a different offsite policy. For
+        example, to allow the domains in
+        :attr:`~scrapy.Spider.allowed_domains` but none of their subdomains:
+
+        .. code-block:: python
+
+            from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware
+            from scrapy.utils.httpobj import urlparse_cached
+
+
+            class RootOnlyOffsiteMiddleware(OffsiteMiddleware):
+                def should_follow(self, request, spider):
+                    return urlparse_cached(request).hostname in spider.allowed_domains
+        """
         self._update_host_regex(spider)
         regex = self.host_regex
         # hostname can be None for wrong urls (like javascript links)
@@ -79,7 +126,6 @@ class OffsiteMiddleware:
         return bool(regex.search(host))
 
     def get_host_regex(self, spider: Spider) -> re.Pattern[str]:
-        """Override this method to implement a different offsite policy"""
         allowed_domains = getattr(spider, "allowed_domains", None)
         if not allowed_domains:
             return re.compile("")  # allow all by default

tests/test_downloadermiddleware_offsite.py

@@ -6,6 +6,7 @@ import pytest
 from scrapy import Request, Spider
 from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware
 from scrapy.exceptions import IgnoreRequest
+from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.test import get_crawler
 
 UNSET = object()
@@ -237,6 +238,21 @@ def test_repeated_offsite_domain():
     assert crawler.stats.get_value("offsite/filtered") == 2
 
 
+def test_should_follow_override():
+    class RootOnlyOffsiteMiddleware(OffsiteMiddleware):
+        def should_follow(self, request: Request, spider: Spider) -> bool:
+            allowed_domains: list[str] = getattr(spider, "allowed_domains", [])
+            return urlparse_cached(request).hostname in allowed_domains
+
+    crawler = get_crawler(Spider)
+    crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
+    mw = RootOnlyOffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(crawler.spider)
+    assert mw.process_request(Request("https://example.com/1")) is None
+    with pytest.raises(IgnoreRequest):
+        mw.process_request(Request("https://www.example.com/1"))
+
+
 def test_ignore_request_reason():
     crawler = get_crawler(Spider)
     crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])