diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 965108214..dec9904d2 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -799,40 +799,9 @@ OffsiteMiddleware .. module:: scrapy.downloadermiddlewares.offsite :synopsis: Offsite Middleware -.. class:: OffsiteMiddleware +.. autoclass:: OffsiteMiddleware - .. versionadded:: 2.11.2 - - Filters out Requests for URLs outside the domains covered by the spider. - - This middleware filters out every request whose host names aren't in the - spider's :attr:`~scrapy.Spider.allowed_domains` attribute. - All subdomains of any domain in the list are also allowed. - E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` - but not ``www2.example.com`` nor ``example.com``. - - When your spider returns a request for a domain not belonging to those - covered by the spider, this middleware will log a debug message similar to - this one:: - - DEBUG: Filtered offsite request to 'offsite.example': - - To avoid filling the log with too much noise, it will only print one of - these messages for each new domain filtered. So, for example, if another - request for ``offsite.example`` is filtered, no log message will be - printed. But if a request for ``other.example`` is filtered, a message - will be printed (but only for the first request filtered). - - If the spider doesn't define an - :attr:`~scrapy.Spider.allowed_domains` attribute, or the - attribute is empty, the offsite middleware will allow all requests. - - .. reqmeta:: allow_offsite - - If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to - ``True`` or :attr:`Request.meta ` has ``allow_offsite`` - set to ``True``, then the OffsiteMiddleware will allow the request even if - its domain is not listed in allowed domains. + .. automethod:: should_follow RedirectMiddleware ------------------ diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py index 28c0e09cb..0d5c9e4c9 100644 --- a/scrapy/downloadermiddlewares/offsite.py +++ b/scrapy/downloadermiddlewares/offsite.py @@ -21,6 +21,36 @@ logger = logging.getLogger(__name__) class OffsiteMiddleware: + """Filter out requests for URLs outside the domains covered by the spider. + + .. versionadded:: 2.11.2 + + A request is allowed if its host name is in the + :attr:`~scrapy.Spider.allowed_domains` attribute of the spider, or is a + subdomain of one of those domains. E.g. ``www.example.org`` also allows + ``bob.www.example.org``, but neither ``www2.example.org`` nor + ``example.org``. See :meth:`should_follow` to use a different policy. + + If the spider does not define :attr:`~scrapy.Spider.allowed_domains`, or + the attribute is empty, every request is allowed. + + Filtered requests are logged as follows:: + + DEBUG: Filtered offsite request to 'offsite.example': + + Only the first request filtered for a given domain is logged, to keep the + log readable. + + .. reqmeta:: allow_offsite + + allow_offsite + ------------- + + Requests with the ``allow_offsite`` :attr:`~scrapy.Request.meta` key set to + ``True``, or with :attr:`~scrapy.Request.dont_filter` set to ``True``, are + allowed regardless of their host name. + """ + crawler: Crawler host_regex: re.Pattern[str] @@ -72,6 +102,23 @@ class OffsiteMiddleware: raise IgnoreRequest(f"Filtered offsite request to {domain!r}") def should_follow(self, request: Request, spider: Spider) -> bool: + """Return ``True`` if *request* is on site, ``False`` if it must be + filtered out. + + Override this method to implement a different offsite policy. For + example, to allow the domains in + :attr:`~scrapy.Spider.allowed_domains` but none of their subdomains: + + .. code-block:: python + + from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware + from scrapy.utils.httpobj import urlparse_cached + + + class RootOnlyOffsiteMiddleware(OffsiteMiddleware): + def should_follow(self, request, spider): + return urlparse_cached(request).hostname in spider.allowed_domains + """ self._update_host_regex(spider) regex = self.host_regex # hostname can be None for wrong urls (like javascript links) @@ -79,7 +126,6 @@ class OffsiteMiddleware: return bool(regex.search(host)) def get_host_regex(self, spider: Spider) -> re.Pattern[str]: - """Override this method to implement a different offsite policy""" allowed_domains = getattr(spider, "allowed_domains", None) if not allowed_domains: return re.compile("") # allow all by default diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index bab89814f..1f91dcd4b 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -6,6 +6,7 @@ import pytest from scrapy import Request, Spider from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware from scrapy.exceptions import IgnoreRequest +from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.test import get_crawler UNSET = object() @@ -237,6 +238,21 @@ def test_repeated_offsite_domain(): assert crawler.stats.get_value("offsite/filtered") == 2 +def test_should_follow_override(): + class RootOnlyOffsiteMiddleware(OffsiteMiddleware): + def should_follow(self, request: Request, spider: Spider) -> bool: + allowed_domains: list[str] = getattr(spider, "allowed_domains", []) + return urlparse_cached(request).hostname in allowed_domains + + crawler = get_crawler(Spider) + crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"]) + mw = RootOnlyOffsiteMiddleware.from_crawler(crawler) + mw.spider_opened(crawler.spider) + assert mw.process_request(Request("https://example.com/1")) is None + with pytest.raises(IgnoreRequest): + mw.process_request(Request("https://www.example.com/1")) + + def test_ignore_request_reason(): crawler = get_crawler(Spider) crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])