diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 84daaaa55..ced481c71 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -273,6 +273,9 @@ OffsiteMiddleware This middleware filters out every request whose host names aren't in the spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute. + All subdomains of any domain in the list are also allowed. + E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org`` + but not ``www2.example.com`` nor ``example.com``. When your spider returns a request for a domain not belonging to those covered by the spider, this middleware will log a debug message similar to diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 5fd187e4e..b700ea0ef 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -76,7 +76,7 @@ scrapy.Spider An optional list of strings containing domains that this spider is allowed to crawl. Requests for URLs not belonging to the domain names - specified in this list won't be followed if + specified in this list (or their subdomains) won't be followed if :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled. .. attribute:: start_urls diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index f88c806d7..37c3a450b 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -16,7 +16,7 @@ class TestOffsiteMiddleware(TestCase): self.mw.spider_opened(self.spider) def _get_spiderargs(self): - return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org']) + return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org', 'scrapy.test.org']) def test_process_spider_output(self): res = Response('http://scrapytest.org') @@ -24,13 +24,16 @@ class TestOffsiteMiddleware(TestCase): onsite_reqs = [Request('http://scrapytest.org/1'), Request('http://scrapy.org/1'), Request('http://sub.scrapy.org/1'), - Request('http://offsite.tld/letmepass', dont_filter=True)] + Request('http://offsite.tld/letmepass', dont_filter=True), + Request('http://scrapy.test.org/')] offsite_reqs = [Request('http://scrapy2.org'), Request('http://offsite.tld/'), Request('http://offsite.tld/scrapytest.org'), Request('http://offsite.tld/rogue.scrapytest.org'), Request('http://rogue.scrapytest.org.haha.com'), - Request('http://roguescrapytest.org')] + Request('http://roguescrapytest.org'), + Request('http://test.org/'), + Request('http://notscrapy.test.org/')] reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider))