From b247fa9982a390e1380c46e390069c6058d97921 Mon Sep 17 00:00:00 2001 From: Ricardo Amendoeira Date: Mon, 29 Mar 2021 01:48:28 +0100 Subject: [PATCH 1/2] Include loading settings in `Running multiple spiders in the same process` section The example in the documentation doesn't take into account the project settings --- docs/topics/practices.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cf1de1bd1..db1ed362e 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -118,6 +118,7 @@ Here is an example that runs multiple spiders simultaneously: :: import scrapy + from scrapy.utils.project import get_project_settings from scrapy.crawler import CrawlerProcess class MySpider1(scrapy.Spider): @@ -128,7 +129,8 @@ Here is an example that runs multiple spiders simultaneously: # Your second spider definition ... - process = CrawlerProcess() + settings = get_project_settings() + process = CrawlerProcess(settings) process.crawl(MySpider1) process.crawl(MySpider2) process.start() # the script will block here until all crawling jobs are finished From 8603f9d7a5524b7709b4e8a8fc04a75a9a4f0ffe Mon Sep 17 00:00:00 2001 From: Ricardo Amendoeira Date: Tue, 6 Apr 2021 20:23:07 +0100 Subject: [PATCH 2/2] Apply changes to other examples in the same section. --- docs/topics/practices.rst | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index db1ed362e..15ac520e2 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -118,8 +118,8 @@ Here is an example that runs multiple spiders simultaneously: :: import scrapy - from scrapy.utils.project import get_project_settings from scrapy.crawler import CrawlerProcess + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -143,6 +143,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: from twisted.internet import reactor from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -153,7 +154,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: ... configure_logging() - runner = CrawlerRunner() + settings = get_project_settings() + runner = CrawlerRunner(settings) runner.crawl(MySpider1) runner.crawl(MySpider2) d = runner.join() @@ -168,6 +170,7 @@ Same example but running the spiders sequentially by chaining the deferreds: from twisted.internet import reactor, defer from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging + from scrapy.utils.project import get_project_settings class MySpider1(scrapy.Spider): # Your first spider definition @@ -178,7 +181,8 @@ Same example but running the spiders sequentially by chaining the deferreds: ... configure_logging() - runner = CrawlerRunner() + settings = get_project_settings() + runner = CrawlerRunner(settings) @defer.inlineCallbacks def crawl():