From e676cd3ce0d488f56498b766912725066bcee4d9 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 22 May 2024 07:55:53 -0300 Subject: [PATCH 01/11] docs: Remove top-level reactor imports from CrawlerProces/CrawlerRunner examples --- docs/topics/practices.rst | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cd359b147..7731180fe 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -92,7 +92,6 @@ reactor after ``MySpider`` has finished running. .. code-block:: python - from twisted.internet import reactor import scrapy from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging @@ -107,6 +106,33 @@ reactor after ``MySpider`` has finished running. runner = CrawlerRunner() d = runner.crawl(MySpider) + from twisted.internet import reactor + + d.addBoth(lambda _: reactor.stop()) + reactor.run() # the script will block here until the crawling is finished + +Same example but using a non-default reactor, is only necessary call ``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. + +.. code-block:: python + + import scrapy + from scrapy.crawler import CrawlerRunner + from scrapy.utils.log import configure_logging + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + from scrapy.utils.reactor import install_reactor + + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + runner = CrawlerRunner() + d = runner.crawl(MySpider) + from twisted.internet import reactor + d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished @@ -151,7 +177,6 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: .. code-block:: python import scrapy - from twisted.internet import reactor from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings @@ -173,6 +198,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: runner.crawl(MySpider1) runner.crawl(MySpider2) d = runner.join() + from twisted.internet import reactor + d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until all crawling jobs are finished @@ -181,7 +208,7 @@ Same example but running the spiders sequentially by chaining the deferreds: .. code-block:: python - from twisted.internet import reactor, defer + from twisted.internet import defer from scrapy.crawler import CrawlerRunner from scrapy.utils.log import configure_logging from scrapy.utils.project import get_project_settings @@ -209,6 +236,8 @@ Same example but running the spiders sequentially by chaining the deferreds: reactor.stop() + from twisted.internet import reactor + crawl() reactor.run() # the script will block here until the last crawl call is finished From 8210fae25a9d812447df617155001b9861e0d834 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 22 May 2024 18:50:50 -0300 Subject: [PATCH 02/11] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 7731180fe..710be7aa2 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -111,7 +111,9 @@ reactor after ``MySpider`` has finished running. d.addBoth(lambda _: reactor.stop()) reactor.run() # the script will block here until the crawling is finished -Same example but using a non-default reactor, is only necessary call ``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. +Same example but using a non-default reactor, it's only necessary call +``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` + already does this automatically. .. code-block:: python From dc6a495fee41949d50178b9e46d6f41e83425ca2 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 22 May 2024 18:51:02 -0300 Subject: [PATCH 03/11] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 710be7aa2..cec098012 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -106,6 +106,7 @@ reactor after ``MySpider`` has finished running. runner = CrawlerRunner() d = runner.crawl(MySpider) + from twisted.internet import reactor d.addBoth(lambda _: reactor.stop()) From 3f66b66e3f645393dbb263a1ec7ab04bdabd74b4 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Wed, 22 May 2024 22:01:55 -0300 Subject: [PATCH 04/11] fix: checks --- docs/topics/practices.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index cec098012..aa81ceea5 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -113,8 +113,7 @@ reactor after ``MySpider`` has finished running. reactor.run() # the script will block here until the crawling is finished Same example but using a non-default reactor, it's only necessary call -``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` - already does this automatically. +``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. .. code-block:: python From e143dc795228424fa98cb40e17b9993617ae61ae Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 22 May 2024 22:26:31 -0300 Subject: [PATCH 05/11] Update tests-macos.yml --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 252176464..95016146e 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,4 +1,4 @@ -name: macOS +name: macOS. on: [push, pull_request] concurrency: From 9d5a0d287b69a69fe34cbe3130438fb36f1f3441 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Wed, 22 May 2024 22:27:07 -0300 Subject: [PATCH 06/11] Retrigger CI --- .github/workflows/tests-macos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index 95016146e..252176464 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,4 +1,4 @@ -name: macOS. +name: macOS on: [push, pull_request] concurrency: From 17e623cf0cfb5c695c43ceb069026d44cb28ca21 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 23 May 2024 07:00:24 -0300 Subject: [PATCH 07/11] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index aa81ceea5..64b3b6e81 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -128,6 +128,7 @@ Same example but using a non-default reactor, it's only necessary call configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + from scrapy.utils.reactor import install_reactor install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") From 8ec67ca230a69effb2e0442fb2a6c06cd6c92adf Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 23 May 2024 07:00:35 -0300 Subject: [PATCH 08/11] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 64b3b6e81..ee484e63f 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -134,6 +134,7 @@ Same example but using a non-default reactor, it's only necessary call install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") runner = CrawlerRunner() d = runner.crawl(MySpider) + from twisted.internet import reactor d.addBoth(lambda _: reactor.stop()) From 62c89aaf056687091235bb846ac565f7c801c359 Mon Sep 17 00:00:00 2001 From: Laerte Pereira <5853172+Laerte@users.noreply.github.com> Date: Thu, 23 May 2024 07:00:45 -0300 Subject: [PATCH 09/11] Update docs/topics/practices.rst Co-authored-by: Andrey Rakhmatullin --- docs/topics/practices.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index ee484e63f..1500011e7 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -202,6 +202,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: runner.crawl(MySpider1) runner.crawl(MySpider2) d = runner.join() + from twisted.internet import reactor d.addBoth(lambda _: reactor.stop()) From 2facdd4fb08ec3edaf1752047dd86d5b565621a1 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Sun, 26 May 2024 19:55:54 -0300 Subject: [PATCH 10/11] Add change reactor test to CrawlerRunner --- .flake8 | 1 + scrapy/crawler.py | 2 ++ tests/CrawlerRunner/change_reactor.py | 31 +++++++++++++++++++++++++++ tests/test_crawler.py | 8 +++++++ 4 files changed, 42 insertions(+) create mode 100644 tests/CrawlerRunner/change_reactor.py diff --git a/.flake8 b/.flake8 index 62ccad9cf..0e43b9b56 100644 --- a/.flake8 +++ b/.flake8 @@ -9,6 +9,7 @@ exclude = per-file-ignores = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file + tests/CrawlerRunner/change_reactor.py:E402 # F401: Module imported but unused scrapy/__init__.py:E402 scrapy/core/downloader/handlers/http.py:F401 diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ccfe78891..4fe5987a7 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -129,6 +129,8 @@ class Crawler: if is_asyncio_reactor_installed() and event_loop: verify_installed_asyncio_event_loop(event_loop) + log_reactor_info() + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py new file mode 100644 index 000000000..b20aa0c7c --- /dev/null +++ b/tests/CrawlerRunner/change_reactor.py @@ -0,0 +1,31 @@ +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + def start_requests(self): + return [] + + +configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"}) + + +from scrapy.utils.reactor import install_reactor + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + +runner = CrawlerRunner() + +d = runner.crawl(NoRequestsSpider) + +from twisted.internet import reactor + +d.addBoth(callback=lambda _: reactor.stop()) +reactor.run() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 989208694..791ea1faa 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -926,3 +926,11 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("INFO: Host: not.a.real.domain", log) self.assertIn("INFO: Type: ", log) self.assertIn("INFO: IP address: 127.0.0.1", log) + + def test_change_default_reactor(self): + log = self.run_script("change_reactor.py") + self.assertIn( + "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", + log, + ) + self.assertIn("DEBUG: Using asyncio event loop", log) From 6cd085785028d97393f26e6fee22e6c03e5c90a8 Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Sun, 26 May 2024 19:57:16 -0300 Subject: [PATCH 11/11] Move path --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 0e43b9b56..cf1a96476 100644 --- a/.flake8 +++ b/.flake8 @@ -9,7 +9,6 @@ exclude = per-file-ignores = # Exclude files that are meant to provide top-level imports # E402: Module level import not at top of file - tests/CrawlerRunner/change_reactor.py:E402 # F401: Module imported but unused scrapy/__init__.py:E402 scrapy/core/downloader/handlers/http.py:F401 @@ -17,6 +16,7 @@ per-file-ignores = scrapy/linkextractors/__init__.py:E402,F401 scrapy/selector/__init__.py:F401 scrapy/spiders/__init__.py:E402,F401 + tests/CrawlerRunner/change_reactor.py:E402 # Issues pending a review: scrapy/utils/url.py:F403,F405