diff --git a/docs/news.rst b/docs/news.rst index 509366c17..2afe318f6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -1,3 +1,25 @@ +.. note:: + .. versionchanged:: VERSION + + The Twisted reactor is now installed when + :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a + :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this, + :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now + honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions + they are silently ignored when set there and you need to set these settings + in some other way. + + +.. note:: + .. versionchanged:: VERSION + + Previously this setting had no effect in a spider + :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but + if you :ref:`run several spiders in one process `, + they must not have different values for this setting, because they will use + a single reactor instance. + + .. _news: Release notes diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 732eba587..1a9d56143 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -193,6 +193,25 @@ Same example but running the spiders sequentially by chaining the deferreds: crawl() reactor.run() # the script will block here until the last crawl call is finished +Different spiders can set different values for the same setting, but when they +run in the same process it may be impossible, by design or because of some +limitations, to use these different values. What happens in practice is +different for different settings: + +* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value + (:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the + default one) cannot be read from the per-spider settings. These are applied + when the :class:`~scrapy.crawler.CrawlerRunner` or + :class:`~scrapy.crawler.CrawlerProcess` object is created. +* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first + available value is used, and if a spider requests a different reactor an + exception will be raised. These are applied when the reactor is installed. +* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the + ones used by the resolver (:setting:`DNSCACHE_ENABLED`, + :setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy) + the first available value is used. These are applied when the reactor is + started. + .. seealso:: :ref:`run-from-script`. .. _distributed-crawls: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 210c1def7..f6c95c502 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1638,10 +1638,9 @@ which raises :exc:`Exception`, becomes:: The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will not attempt to install any specific reactor, and the -default reactor defined by Twisted for the current platform will be used. This -is to maintain backward compatibility and avoid possible problems caused by -using a non-default reactor. +means that Scrapy will install the default reactor defined by Twisted for the +current platform. This is to maintain backward compatibility and avoid possible +problems caused by using a non-default reactor. For additional information, see :doc:`core/howto/choosing-reactor`. diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index d1944df3d..de81986d8 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -75,6 +75,6 @@ class Command(ScrapyCommand): def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, - kwargs={'stop_after_crawl': False}) + kwargs={'stop_after_crawl': False, 'install_signal_handlers': False}) t.daemon = True t.start() diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 578016536..a638254f1 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -25,6 +25,7 @@ from scrapy.utils.log import ( configure_logging, get_scrapy_root_handler, install_scrapy_root_handler, + log_reactor_info, log_scrapy_info, LogCounterHandler, ) @@ -38,7 +39,7 @@ logger = logging.getLogger(__name__) class Crawler: - def __init__(self, spidercls, settings=None): + def __init__(self, spidercls, settings=None, init_reactor: bool = False): if isinstance(spidercls, Spider): raise ValueError('The spidercls argument must be a class, not an object') @@ -69,6 +70,20 @@ class Crawler: lf_cls = load_object(self.settings['LOG_FORMATTER']) self.logformatter = lf_cls.from_crawler(self) + + reactor_class = self.settings.get("TWISTED_REACTOR") + if init_reactor: + # this needs to be done after the spider settings are merged, + # but before something imports twisted.internet.reactor + if reactor_class: + install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"]) + else: + from twisted.internet import default + default.install() + log_reactor_info() + if reactor_class: + verify_installed_reactor(reactor_class) + self.extensions = ExtensionManager.from_crawler(self) self.settings.freeze() @@ -153,7 +168,6 @@ class CrawlerRunner: self._crawlers = set() self._active = set() self.bootstrap_failed = False - self._handle_twisted_reactor() @property def spiders(self): @@ -247,10 +261,6 @@ class CrawlerRunner: while self._active: yield defer.DeferredList(self._active) - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - verify_installed_reactor(self.settings["TWISTED_REACTOR"]) - class CrawlerProcess(CrawlerRunner): """ @@ -278,7 +288,6 @@ class CrawlerProcess(CrawlerRunner): def __init__(self, settings=None, install_root_handler=True): super().__init__(settings) - install_shutdown_handlers(self._signal_shutdown) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) @@ -298,7 +307,12 @@ class CrawlerProcess(CrawlerRunner): {'signame': signame}) reactor.callFromThread(self._stop_reactor) - def start(self, stop_after_crawl=True): + def _create_crawler(self, spidercls): + if isinstance(spidercls, str): + spidercls = self.spider_loader.load(spidercls) + return Crawler(spidercls, self.settings, init_reactor=True) + + def start(self, stop_after_crawl=True, install_signal_handlers=True): """ This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache @@ -309,6 +323,9 @@ class CrawlerProcess(CrawlerRunner): :param bool stop_after_crawl: stop or not the reactor when all crawlers have finished + + :param bool install_signal_handlers: whether to install the shutdown + handlers (default: True) """ from twisted.internet import reactor if stop_after_crawl: @@ -318,6 +335,8 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) + if install_signal_handlers: + install_shutdown_handlers(self._signal_shutdown) resolver_class = load_object(self.settings["DNS_RESOLVER"]) resolver = create_instance(resolver_class, self.settings, self, reactor=reactor) resolver.install_on_reactor() @@ -337,8 +356,3 @@ class CrawlerProcess(CrawlerRunner): reactor.stop() except RuntimeError: # raised if already stopped or in shutdown stage pass - - def _handle_twisted_reactor(self): - if self.settings.get("TWISTED_REACTOR"): - install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"]) - super()._handle_twisted_reactor() diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 0441c0358..78e302d19 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -143,7 +143,7 @@ def _get_handler(settings): return handler -def log_scrapy_info(settings): +def log_scrapy_info(settings: Settings) -> None: logger.info("Scrapy %(version)s started (bot: %(bot)s)", {'version': scrapy.__version__, 'bot': settings['BOT_NAME']}) versions = [ @@ -152,6 +152,9 @@ def log_scrapy_info(settings): if name != "Scrapy" ] logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)}) + + +def log_reactor_info() -> None: from twisted.internet import reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) from twisted.internet import asyncioreactor diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings.py b/tests/CrawlerProcess/twisted_reactor_custom_settings.py new file mode 100644 index 000000000..56304bd23 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings.py @@ -0,0 +1,14 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py new file mode 100644 index 000000000..3f219098c --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class SelectReactorSpider(scrapy.Spider): + name = 'select_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", + } + + +class AsyncioReactorSpider(scrapy.Spider): + name = 'asyncio_reactor' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(SelectReactorSpider) +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py new file mode 100644 index 000000000..1f5a44010 --- /dev/null +++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py @@ -0,0 +1,21 @@ +import scrapy +from scrapy.crawler import CrawlerProcess + + +class AsyncioReactorSpider1(scrapy.Spider): + name = 'asyncio_reactor1' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + +class AsyncioReactorSpider2(scrapy.Spider): + name = 'asyncio_reactor2' + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = CrawlerProcess() +process.crawl(AsyncioReactorSpider1) +process.crawl(AsyncioReactorSpider2) +process.start() diff --git a/tests/test_commands.py b/tests/test_commands.py index 81d1a1cab..7473b53df 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -762,6 +762,7 @@ class MySpider(scrapy.Spider): self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log) +@skipIf(platform.system() != 'Windows', "Windows required for .pyw files") class WindowsRunSpiderCommandTest(RunSpiderCommandTest): spider_filename = 'myspider.pyw' @@ -774,35 +775,27 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest): self.assertIn("start_requests", log) self.assertIn("badspider.pyw", log) - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_run_good_spider(self): super().test_run_good_spider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider(self): super().test_runspider() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_dnscache_disabled(self): super().test_runspider_dnscache_disabled() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_level(self): super().test_runspider_log_level() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_log_short_names(self): super().test_runspider_log_short_names() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_runspider_no_spider_found(self): super().test_runspider_no_spider_found() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_output(self): super().test_output() - @skipIf(platform.system() != 'Windows', "Windows required for .pyw files") def test_overwrite_output(self): super().test_overwrite_output() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 7bc4fba40..8f6227109 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -6,7 +6,6 @@ import sys import warnings from pytest import raises, mark -from testfixtures import LogCapture from twisted import version as twisted_version from twisted.internet import defer from twisted.python.versions import Version @@ -270,6 +269,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertEqual(runner.bootstrap_failed, True) + @defer.inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): if self.reactor_pytest == 'asyncio': CrawlerRunner(settings={ @@ -278,32 +278,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase): else: msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" with self.assertRaisesRegex(Exception, msg): - CrawlerRunner(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - - @defer.inlineCallbacks - def test_crawler_process_asyncio_enabled_true(self): - with LogCapture(level=logging.DEBUG) as log: - if self.reactor_pytest == 'asyncio': - runner = CrawlerProcess(settings={ + runner = CrawlerRunner(settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", }) yield runner.crawl(NoRequestsSpider) - self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) - else: - msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)" - with self.assertRaisesRegex(Exception, msg): - runner = CrawlerProcess(settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - }) - - @defer.inlineCallbacks - def test_crawler_process_asyncio_enabled_false(self): - runner = CrawlerProcess(settings={"TWISTED_REACTOR": None}) - with LogCapture(level=logging.DEBUG) as log: - yield runner.crawl(NoRequestsSpider) - self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) class ScriptRunnerMixin: @@ -372,6 +350,21 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + def test_reactor_asyncio_custom_settings(self): + log = self.run_script("twisted_reactor_custom_settings.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + def test_reactor_asyncio_custom_settings_same(self): + log = self.run_script("twisted_reactor_custom_settings_same.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + def test_reactor_asyncio_custom_settings_conflict(self): + log = self.run_script("twisted_reactor_custom_settings_conflict.py") + self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log) + self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log) + @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly') @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows') @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')