diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 5937c5860..59c821175 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -29,6 +29,7 @@ https://quotes.toscrape.com, following the pagination: class QuotesSpider(scrapy.Spider): name = "quotes" + allowed_domains = ["quotes.toscrape.com"] start_urls = [ "https://quotes.toscrape.com/tag/humor/", ] diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 95c80d5dc..531f6324d 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -286,6 +286,7 @@ Spiders can access arguments in their `__init__` methods: class MySpider(scrapy.Spider): name = "myspider" + allowed_domains = ["www.example.com"] def __init__(self, category=None, *args, **kwargs): super().__init__(*args, **kwargs) @@ -303,6 +304,7 @@ The above example can also be written as follows: class MySpider(scrapy.Spider): name = "myspider" + allowed_domains = ["www.example.com"] async def start(self): yield scrapy.Request(f"http://www.example.com/categories/{self.category}") @@ -357,6 +359,7 @@ is automatically converted to an integer: class BookSpider(Args[MyParams], scrapy.Spider): name = "bookspider" + allowed_domains = ["books.toscrape.com"] start_urls = ["http://books.toscrape.com/catalogue"] async def start(self): @@ -924,6 +927,7 @@ Simplest example: process all urls discovered through sitemaps using the class MySpider(SitemapSpider): + allowed_domains = ["www.example.com"] sitemap_urls = ["http://www.example.com/sitemap.xml"] def parse(self, response): @@ -938,6 +942,7 @@ callback: class MySpider(SitemapSpider): + allowed_domains = ["www.example.com"] sitemap_urls = ["http://www.example.com/sitemap.xml"] sitemap_rules = [ ("/product/", "parse_product"), @@ -959,6 +964,7 @@ whose url contains ``/sitemap_shop``: class MySpider(SitemapSpider): + allowed_domains = ["www.example.com"] sitemap_urls = ["http://www.example.com/robots.txt"] sitemap_rules = [ ("/shop/", "parse_shop"), @@ -977,6 +983,7 @@ Combine SitemapSpider with other sources of urls: class MySpider(SitemapSpider): + allowed_domains = ["www.example.com"] sitemap_urls = ["http://www.example.com/robots.txt"] sitemap_rules = [ ("/shop/", "parse_shop"),