From e676cd3ce0d488f56498b766912725066bcee4d9 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 22 May 2024 07:55:53 -0300 Subject: [PATCH] docs: Remove top-level reactor imports from CrawlerProces/CrawlerRunner examples --- docs/topics/practices.rst | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cd359b147..7731180fe 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -92,7 +92,6 @@ reactor after ``MySpider`` has finished running. .. code-block:: python - from twisted.internet import reactor import scrapy from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging @@ -107,6 +106,33 @@ reactor after ``MySpider`` has finished running. runner = CrawlerRunner() d = runner.crawl(MySpider) + from twisted.internet import reactor + + d.addBoth(lambda _: reactor.stop()) + reactor.run() # the script will block here until the crawling is finished + +Same example but using a non-default reactor, is only necessary call ``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. + +.. code-block:: python + + import scrapy + from scrapy.crawler import CrawlerRunner + from scrapy.utils.log import configure_logging + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + from scrapy.utils.reactor import install_reactor + + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + runner = CrawlerRunner() + d = runner.crawl(MySpider) + from twisted.internet import reactor + d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished @@ -151,7 +177,6 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: .. code-block:: python import scrapy - from twisted.internet import reactor from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings @@ -173,6 +198,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: runner.crawl(MySpider1) runner.crawl(MySpider2) d = runner.join() + from twisted.internet import reactor + d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until all crawling jobs are finished @@ -181,7 +208,7 @@ Same example but running the spiders sequentially by chaining the deferreds: .. code-block:: python - from twisted.internet import reactor, defer + from twisted.internet import defer from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings @@ -209,6 +236,8 @@ Same example but running the spiders sequentially by chaining the deferreds: reactor.stop() + from twisted.internet import reactor + crawl() reactor.run() # the script will block here until the last crawl call is finished