From fc32e89ac92edcf095a7ea45627c36407eb48f4a Mon Sep 17 00:00:00 2001 From: Diogo Castro Date: Fri, 19 Jun 2026 00:45:35 -0300 Subject: [PATCH] feat: Adding documentation to feature --- docs/topics/downloader-middleware.rst | 15 +++++++++++++++ docs/topics/spiders.rst | 12 ++++++++++++ tests/test_downloadermiddleware_offsite.py | 4 +--- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0c1af5276..f6fafbb52 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -856,6 +856,21 @@ OffsiteMiddleware :attr:`~scrapy.Spider.allowed_domains` attribute, or the attribute is empty, the offsite middleware will allow all requests. + If the spider defines a :attr:`~scrapy.Spider.disallowed_domains` + attribute, any request whose host name matches one of the domains in + that list (or their subdomains) will be filtered out, regardless of + :attr:`~scrapy.Spider.allowed_domains`. This is useful when you want + to allow most domains but block a few specific ones. + + If a request matches both :attr:`~scrapy.Spider.allowed_domains` and + :attr:`~scrapy.Spider.disallowed_domains`, it will be filtered out + (i.e. :attr:`~scrapy.Spider.disallowed_domains` takes precedence). + + Both :attr:`~scrapy.Spider.allowed_domains` and + :attr:`~scrapy.Spider.disallowed_domains` must contain valid domain + names only (not URLs or domains with ports). Invalid entries will cause + the spider to close with reason ``invalid_domain_configuration``. + .. reqmeta:: allow_offsite If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 506daf930..56f41590a 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -74,6 +74,18 @@ scrapy.Spider Let's say your target url is ``https://www.example.com/1.html``, then add ``'example.com'`` to the list. + .. attribute:: disallowed_domains + + An optional list of strings containing domains that this spider is + not allowed to crawl. Requests for URLs belonging to the domain names + specified in this list (or their subdomains) will be filtered out when + :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is + enabled. + + This is useful when you want to allow all domains except a few + specific ones. For example, to block requests to ``ads.example.com``, + add ``'ads.example.com'`` to the list. + .. autoattribute:: start_urls .. attribute:: custom_settings diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py index 1d459c8cb..0d7e8982b 100644 --- a/tests/test_downloadermiddleware_offsite.py +++ b/tests/test_downloadermiddleware_offsite.py @@ -135,9 +135,7 @@ def test_invalid_domains_closes_spider(caplog): mock_engine = AsyncMock() crawler.engine = mock_engine with ( - patch( - "scrapy.downloadermiddlewares.offsite._schedule_coro" - ) as mock_schedule, + patch("scrapy.downloadermiddlewares.offsite._schedule_coro") as mock_schedule, caplog.at_level(logging.ERROR), ): mw.spider_opened(crawler.spider)