From defe7a4382e88715df927ab4072dc88865a6f899 Mon Sep 17 00:00:00 2001 From: Cj Malone Date: Mon, 2 Jun 2025 12:59:21 +0100 Subject: [PATCH 1/2] [doc] Add allowed_domains to example spiders --- docs/intro/overview.rst | 1 + docs/topics/debug.rst | 1 + docs/topics/logging.rst | 2 ++ docs/topics/request-response.rst | 3 +++ docs/topics/settings.rst | 3 +++ docs/topics/shell.rst | 1 + docs/topics/signals.rst | 1 + docs/topics/spiders.rst | 6 ++++++ 8 files changed, 18 insertions(+) diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index d05e46551..427f70ed7 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -29,6 +29,7 @@ https://quotes.toscrape.com, following the pagination: class QuotesSpider(scrapy.Spider): name = "quotes" + allowed_domains = ["quotes.toscrape.com"] start_urls = [ "https://quotes.toscrape.com/tag/humor/", ] diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 988e37bbd..9986bf0f2 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -16,6 +16,7 @@ Consider the following Scrapy spider below: class MySpider(scrapy.Spider): name = "myspider" + allowed_domains = ["example.com"] start_urls = ( "http://example.com/page1", "http://example.com/page2", diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index a398d6c83..4f6a0be8d 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -118,6 +118,7 @@ instance, which can be accessed and used like this: class MySpider(scrapy.Spider): name = "myspider" + allowed_domains = ["scrapy.org"] start_urls = ["https://scrapy.org"] def parse(self, response): @@ -136,6 +137,7 @@ Python logger you want. For example: class MySpider(scrapy.Spider): name = "myspider" + allowed_domains = ["scrapy.org"] start_urls = ["https://scrapy.org"] def parse(self, response): diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 6ca0973d8..e99dca15f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -344,6 +344,7 @@ errors if needed: class ErrbackSpider(scrapy.Spider): name = "errback_example" + allowed_domains = ["www.httpbin.org", "example.invalid"] start_urls = [ "http://www.httpbin.org/", # HTTP 200 expected "http://www.httpbin.org/status/404", # Not found error @@ -712,6 +713,7 @@ signals will stop the download of a given response. See the following example: class StopSpider(scrapy.Spider): name = "stop" + allowed_domains = ["docs.scrapy.org"] start_urls = ["https://docs.scrapy.org/en/latest/"] @classmethod @@ -889,6 +891,7 @@ method for this job. Here's an example spider which uses it: class LoginSpider(scrapy.Spider): name = "example.com" + allowed_domains = ["www.example.com"] start_urls = ["http://www.example.com/users/login.php"] def parse(self, response): diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 65f2e5ebd..8430624a5 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -219,6 +219,7 @@ In a spider, settings are available through ``self.settings``: class MySpider(scrapy.Spider): name = "myspider" + allowed_domains = ["example.com"] start_urls = ["http://example.com"] def parse(self, response): @@ -2012,6 +2013,7 @@ In order to use the reactor installed by Scrapy: class QuotesSpider(scrapy.Spider): name = "quotes" + allowed_domains = ["quotes.toscrape.com"] def __init__(self, *args, **kwargs): self.timeout = int(kwargs.pop("timeout", "60")) @@ -2041,6 +2043,7 @@ which raises an exception, becomes: class QuotesSpider(scrapy.Spider): name = "quotes" + allowed_domains = ["quotes.toscrape.com"] def __init__(self, *args, **kwargs): self.timeout = int(kwargs.pop("timeout", "60")) diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 4898843e4..e76e8400d 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -253,6 +253,7 @@ Here's an example of how you would call it from your spider: class MySpider(scrapy.Spider): name = "myspider" + allowed_domains = ["example.com", "example.org", "example.net"] start_urls = [ "http://example.com", "http://example.org", diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index aa27e62dd..6b8b717bd 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -66,6 +66,7 @@ Let's take an example using :ref:`coroutines `: class SignalSpider(scrapy.Spider): name = "signals" + allowed_domains = ["quotes.toscrape.com"] start_urls = ["https://quotes.toscrape.com/page/1/"] @classmethod diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 8240d5d4b..531b32541 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -312,6 +312,7 @@ Spiders can access arguments in their `__init__` methods: class MySpider(scrapy.Spider): name = "myspider" + allowed_domains = ["www.example.com"] def __init__(self, category=None, *args, **kwargs): super(MySpider, self).__init__(*args, **kwargs) @@ -329,6 +330,7 @@ The above example can also be written as follows: class MySpider(scrapy.Spider): name = "myspider" + allowed_domains = ["www.example.com"] async def start(self): yield scrapy.Request(f"http://www.example.com/categories/{self.category}") @@ -867,6 +869,7 @@ Simplest example: process all urls discovered through sitemaps using the class MySpider(SitemapSpider): + allowed_domains = ["www.example.com"] sitemap_urls = ["http://www.example.com/sitemap.xml"] def parse(self, response): @@ -881,6 +884,7 @@ callback: class MySpider(SitemapSpider): + allowed_domains = ["www.example.com"] sitemap_urls = ["http://www.example.com/sitemap.xml"] sitemap_rules = [ ("/product/", "parse_product"), @@ -902,6 +906,7 @@ whose url contains ``/sitemap_shop``: class MySpider(SitemapSpider): + allowed_domains = ["www.example.com"] sitemap_urls = ["http://www.example.com/robots.txt"] sitemap_rules = [ ("/shop/", "parse_shop"), @@ -919,6 +924,7 @@ Combine SitemapSpider with other sources of urls: class MySpider(SitemapSpider): + allowed_domains = ["www.example.com"] sitemap_urls = ["http://www.example.com/robots.txt"] sitemap_rules = [ ("/shop/", "parse_shop"), From e0db7f16130bef01b8db1186ceb1ddec93036c96 Mon Sep 17 00:00:00 2001 From: Cj Malone Date: Tue, 4 Aug 2026 16:31:00 +0100 Subject: [PATCH 2/2] Trim examples --- docs/topics/debug.rst | 1 - docs/topics/logging.rst | 2 -- docs/topics/request-response.rst | 4 ---- docs/topics/settings.rst | 3 --- docs/topics/shell.rst | 1 - docs/topics/signals.rst | 2 -- 6 files changed, 13 deletions(-) diff --git a/docs/topics/debug.rst b/docs/topics/debug.rst index 9986bf0f2..988e37bbd 100644 --- a/docs/topics/debug.rst +++ b/docs/topics/debug.rst @@ -16,7 +16,6 @@ Consider the following Scrapy spider below: class MySpider(scrapy.Spider): name = "myspider" - allowed_domains = ["example.com"] start_urls = ( "http://example.com/page1", "http://example.com/page2", diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst index 22d5da25f..b486eab63 100644 --- a/docs/topics/logging.rst +++ b/docs/topics/logging.rst @@ -113,7 +113,6 @@ instance, which can be accessed and used like this: class MySpider(scrapy.Spider): name = "myspider" - allowed_domains = ["scrapy.org"] start_urls = ["https://scrapy.org"] def parse(self, response): @@ -132,7 +131,6 @@ Python logger you want. For example: class MySpider(scrapy.Spider): name = "myspider" - allowed_domains = ["scrapy.org"] start_urls = ["https://scrapy.org"] def parse(self, response): diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index cef54c946..75158440b 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -325,7 +325,6 @@ credentials: class LoginSpider(scrapy.Spider): name = "example.com" - allowed_domains = ["www.example.com"] start_urls = ["http://www.example.com/users/login.php"] def parse(self, response): @@ -567,7 +566,6 @@ the crawl: class BookSpider(Spider): name = "books" - allowed_domains = ["books.toscrape.com"] async def start(self): yield Request("https://books.toscrape.com/", callback=self.parse_home) @@ -785,7 +783,6 @@ errors if needed: class ErrbackSpider(Spider): name = "errback_example" - allowed_domains = ["www.httpbin.org", "example.invalid"] start_urls = [ "http://www.httpbin.org/", # HTTP 200 expected "http://www.httpbin.org/status/404", # Not found error @@ -1051,7 +1048,6 @@ signals will stop the download of a given response. See the following example: class StopSpider(scrapy.Spider): name = "stop" - allowed_domains = ["docs.scrapy.org"] start_urls = ["https://docs.scrapy.org/en/latest/"] @classmethod diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 3c624ba84..65ee77258 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -217,7 +217,6 @@ In a spider, settings are available through ``self.settings``: class MySpider(scrapy.Spider): name = "myspider" - allowed_domains = ["example.com"] start_urls = ["http://example.com"] def parse(self, response): @@ -2231,7 +2230,6 @@ In order to use the reactor installed by Scrapy: class QuotesSpider(scrapy.Spider): name = "quotes" - allowed_domains = ["quotes.toscrape.com"] def __init__(self, *args, **kwargs): self.timeout = int(kwargs.pop("timeout", "60")) @@ -2261,7 +2259,6 @@ which raises an exception, becomes: class QuotesSpider(scrapy.Spider): name = "quotes" - allowed_domains = ["quotes.toscrape.com"] def __init__(self, *args, **kwargs): self.timeout = int(kwargs.pop("timeout", "60")) diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index fafe8ee19..6f7e67cf9 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -262,7 +262,6 @@ Here's an example of how you would call it from your spider: class MySpider(scrapy.Spider): name = "myspider" - allowed_domains = ["example.com", "example.org", "example.net"] start_urls = [ "http://example.com", "http://example.org", diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index f17c7a0c7..8f0f4d8e9 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -26,7 +26,6 @@ Here is a simple example showing how you can catch signals and perform some acti class DmozSpider(Spider): name = "dmoz" - allowed_domains = ["dmoz.org"] start_urls = [ "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/", "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/", @@ -68,7 +67,6 @@ Let's take an example using :ref:`coroutines `: class SignalSpider(scrapy.Spider): name = "signals" - allowed_domains = ["quotes.toscrape.com"] start_urls = ["https://quotes.toscrape.com/page/1/"] @classmethod