Merge pull request #6374 from Laerte/master

docs: Remove top-level reactor imports from CrawlerProces/CrawlerRunner examples

.flake8

@@ -16,6 +16,7 @@ per-file-ignores =
     scrapy/linkextractors/__init__.py:E402,F401
     scrapy/selector/__init__.py:F401
     scrapy/spiders/__init__.py:E402,F401
+    tests/CrawlerRunner/change_reactor.py:E402
 
     # Issues pending a review:
     scrapy/utils/url.py:F403,F405

docs/topics/practices.rst

@@ -92,7 +92,6 @@ reactor after ``MySpider`` has finished running.
 
 .. code-block:: python
 
-    from twisted.internet import reactor
     import scrapy
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.log import configure_logging
@@ -107,6 +106,37 @@ reactor after ``MySpider`` has finished running.
     runner = CrawlerRunner()
 
     d = runner.crawl(MySpider)
+
+    from twisted.internet import reactor
+
+    d.addBoth(lambda _: reactor.stop())
+    reactor.run()  # the script will block here until the crawling is finished
+
+Same example but using a non-default reactor, it's only necessary call
+``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically.
+
+.. code-block:: python
+
+    import scrapy
+    from scrapy.crawler import CrawlerRunner
+    from scrapy.utils.log import configure_logging
+
+
+    class MySpider(scrapy.Spider):
+        # Your spider definition
+        ...
+
+
+    configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"})
+
+    from scrapy.utils.reactor import install_reactor
+
+    install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+    runner = CrawlerRunner()
+    d = runner.crawl(MySpider)
+
+    from twisted.internet import reactor
+
     d.addBoth(lambda _: reactor.stop())
     reactor.run()  # the script will block here until the crawling is finished
 
@@ -151,7 +181,6 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
 .. code-block:: python
 
     import scrapy
-    from twisted.internet import reactor
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.log import configure_logging
     from scrapy.utils.project import get_project_settings
@@ -173,6 +202,9 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
     runner.crawl(MySpider1)
     runner.crawl(MySpider2)
     d = runner.join()
+
+    from twisted.internet import reactor
+
     d.addBoth(lambda _: reactor.stop())
 
     reactor.run()  # the script will block here until all crawling jobs are finished
@@ -181,7 +213,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
 
 .. code-block:: python
 
-    from twisted.internet import reactor, defer
+    from twisted.internet import defer
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.log import configure_logging
     from scrapy.utils.project import get_project_settings
@@ -209,6 +241,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
         reactor.stop()
 
 
+    from twisted.internet import reactor
+
     crawl()
     reactor.run()  # the script will block here until the last crawl call is finished
 

scrapy/crawler.py

@@ -129,6 +129,8 @@ class Crawler:
             if is_asyncio_reactor_installed() and event_loop:
                 verify_installed_asyncio_event_loop(event_loop)
 
+            log_reactor_info()
+
         self.extensions = ExtensionManager.from_crawler(self)
         self.settings.freeze()
 

tests/CrawlerRunner/change_reactor.py

@@ -0,0 +1,31 @@
+from scrapy import Spider
+from scrapy.crawler import CrawlerRunner
+from scrapy.utils.log import configure_logging
+
+
+class NoRequestsSpider(Spider):
+    name = "no_request"
+
+    custom_settings = {
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+
+    def start_requests(self):
+        return []
+
+
+configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"})
+
+
+from scrapy.utils.reactor import install_reactor
+
+install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+
+runner = CrawlerRunner()
+
+d = runner.crawl(NoRequestsSpider)
+
+from twisted.internet import reactor
+
+d.addBoth(callback=lambda _: reactor.stop())
+reactor.run()

tests/test_crawler.py

@@ -926,3 +926,11 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("INFO: Host: not.a.real.domain", log)
         self.assertIn("INFO: Type: <class 'ipaddress.IPv4Address'>", log)
         self.assertIn("INFO: IP address: 127.0.0.1", log)
+
+    def test_change_default_reactor(self):
+        log = self.run_script("change_reactor.py")
+        self.assertIn(
+            "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+            log,
+        )
+        self.assertIn("DEBUG: Using asyncio event loop", log)