diff --git a/conftest.py b/conftest.py index 18132b7e6..ed7d14166 100644 --- a/conftest.py +++ b/conftest.py @@ -19,6 +19,8 @@ collect_ignore = [ "tests/mockserver.py", "tests/pipelines.py", "tests/spiders.py", + # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess + *_py_files("tests/AsyncCrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess *_py_files("tests/AsyncCrawlerRunner"), # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 3e7bc45c5..b11de2914 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -99,13 +99,11 @@ how you :ref:`configure the downloader middlewares provided while constructing the crawler, and it is created after the arguments given in the :meth:`crawl` method. - .. method:: crawl(*args, **kwargs) + .. automethod:: crawl_async - Starts the crawler by instantiating its spider class with the given - ``args`` and ``kwargs`` arguments, while setting the execution engine in - motion. Should be called only once. + .. automethod:: crawl - Returns a deferred that is fired when the crawl is finished. + .. automethod:: stop_async .. automethod:: stop @@ -115,6 +113,11 @@ how you :ref:`configure the downloader middlewares .. autoclass:: CrawlerRunner :members: +.. autoclass:: AsyncCrawlerProcess + :show-inheritance: + :members: + :inherited-members: + .. autoclass:: CrawlerProcess :show-inheritance: :members: diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 473ef7bfa..ad5c71fbf 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -20,7 +20,8 @@ To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, which is the default value. -If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to +If you are using :class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.CrawlerRunner`, you also need to install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` reactor manually. You can do that using :func:`~scrapy.utils.reactor.install_reactor`: @@ -169,4 +170,8 @@ Switching to a non-asyncio reactor If for some reason your code doesn't work with the asyncio reactor, you can use a different reactor by setting the :setting:`TWISTED_REACTOR` setting to its import path (e.g. ``'twisted.internet.epollreactor.EPollReactor'``) or to -``None``, which will use the default reactor for your platform. +``None``, which will use the default reactor for your platform. If you are +using :class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.AsyncCrawlerProcess` you also need to switch to their +Deferred-based counterparts: :class:`~scrapy.crawler.CrawlerRunner` or +:class:`~scrapy.crawler.CrawlerProcess` respectively. diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 18005aaf2..56177ba4e 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -21,16 +21,21 @@ Remember that Scrapy is built on top of the Twisted asynchronous networking library, so you need to run it inside the Twisted reactor. The first utility you can use to run your spiders is -:class:`scrapy.crawler.CrawlerProcess`. This class will start a Twisted reactor -for you, configuring the logging and setting shutdown handlers. This class is -the one used by all Scrapy commands. +:class:`scrapy.crawler.AsyncCrawlerProcess` or +:class:`scrapy.crawler.CrawlerProcess`. These classes will start a Twisted +reactor for you, configuring the logging and setting shutdown handlers. These +classes are the ones used by all Scrapy commands. They have similar +functionality, differing in their asynchronous API style: +:class:`~scrapy.crawler.AsyncCrawlerProcess` returns coroutines from its +asynchronous methods while :class:`~scrapy.crawler.CrawlerProcess` returns +:class:`~twisted.internet.defer.Deferred` objects. Here's an example showing how to run a single spider with it. .. code-block:: python import scrapy - from scrapy.crawler import CrawlerProcess + from scrapy.crawler import AsyncCrawlerProcess class MySpider(scrapy.Spider): @@ -38,7 +43,7 @@ Here's an example showing how to run a single spider with it. ... - process = CrawlerProcess( + process = AsyncCrawlerProcess( settings={ "FEEDS": { "items.json": {"format": "json"}, @@ -49,53 +54,57 @@ Here's an example showing how to run a single spider with it. process.crawl(MySpider) process.start() # the script will block here until the crawling is finished -Define settings within dictionary in CrawlerProcess. Make sure to check :class:`~scrapy.crawler.CrawlerProcess` +You can define :ref:`settings ` within the dictionary passed +to :class:`~scrapy.crawler.AsyncCrawlerProcess`. Make sure to check the +:class:`~scrapy.crawler.AsyncCrawlerProcess` documentation to get acquainted with its usage details. If you are inside a Scrapy project there are some additional helpers you can use to import those components within the project. You can automatically import -your spiders passing their name to :class:`~scrapy.crawler.CrawlerProcess`, and -use ``get_project_settings`` to get a :class:`~scrapy.settings.Settings` -instance with your project settings. +your spiders passing their name to +:class:`~scrapy.crawler.AsyncCrawlerProcess`, and use +:func:`scrapy.utils.project.get_project_settings` to get a +:class:`~scrapy.settings.Settings` instance with your project settings. What follows is a working example of how to do that, using the `testspiders`_ project as example. .. code-block:: python - from scrapy.crawler import CrawlerProcess + from scrapy.crawler import AsyncCrawlerProcess from scrapy.utils.project import get_project_settings - process = CrawlerProcess(get_project_settings()) + process = AsyncCrawlerProcess(get_project_settings()) # 'followall' is the name of one of the spiders of the project. process.crawl("followall", domain="scrapy.org") process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling -process: :class:`scrapy.crawler.AsyncCrawlerRunner` and +process: :class:`scrapy.crawler.AsyncCrawlerRunner` or :class:`scrapy.crawler.CrawlerRunner`. These classes are thin wrappers that encapsulate some simple helpers to run multiple crawlers, but they won't -start or interfere with existing reactors in any way. They have similar -functionality, differing in their asynchronous API style: -:class:`~scrapy.crawler.AsyncCrawlerRunner` returns coroutines from its -asynchronous methods while :class:`~scrapy.crawler.CrawlerRunner` returns -:class:`~twisted.internet.defer.Deferred` objects. +start or interfere with existing reactors in any way. Just like +:class:`scrapy.crawler.AsyncCrawlerProcess` and +:class:`scrapy.crawler.CrawlerProcess` they differ in their asynchronous API +style. When using these classes the reactor should be explicitly run after scheduling your spiders. It's recommended that you use :class:`~scrapy.crawler.AsyncCrawlerRunner` or :class:`~scrapy.crawler.CrawlerRunner` instead of +:class:`~scrapy.crawler.AsyncCrawlerProcess` or :class:`~scrapy.crawler.CrawlerProcess` if your application is already using Twisted and you want to run Scrapy in the same reactor. If you want to stop the reactor or run any other code right after the spider -finishes you can do that after the :meth:`AsyncCrawlerRunner.crawl() -` coroutine completes (or the Deferred -returned from :meth:`CrawlerRunner.crawl() +finishes you can do that after the task returned from +:meth:`AsyncCrawlerRunner.crawl() ` +completes (or the Deferred returned from :meth:`CrawlerRunner.crawl() ` fires). In the simplest case you can also use :func:`twisted.internet.task.react` to start and stop the reactor, though -it may be easier to just use :class:`~scrapy.crawler.CrawlerProcess` instead. +it may be easier to just use :class:`~scrapy.crawler.AsyncCrawlerProcess` or +:class:`~scrapy.crawler.CrawlerProcess` instead. Here's an example of using :class:`~scrapy.crawler.AsyncCrawlerRunner` together with simple reactor management code: @@ -171,7 +180,7 @@ Here is an example that runs multiple spiders simultaneously: .. code-block:: python import scrapy - from scrapy.crawler import CrawlerProcess + from scrapy.crawler import AsyncCrawlerProcess from scrapy.utils.project import get_project_settings @@ -186,7 +195,7 @@ Here is an example that runs multiple spiders simultaneously: settings = get_project_settings() - process = CrawlerProcess(settings) + process = AsyncCrawlerProcess(settings) process.crawl(MySpider1) process.crawl(MySpider2) process.start() # the script will block here until all crawling jobs are finished diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 0f81a0c0a..68c5079cf 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -330,7 +330,8 @@ exception is raised. These settings are: -- :setting:`ASYNCIO_EVENT_LOOP` +- :setting:`ASYNCIO_EVENT_LOOP` (not possible to set per-spider when using + :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below) - :setting:`DNS_RESOLVER` and settings used by the corresponding component, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE` @@ -338,12 +339,25 @@ These settings are: - :setting:`REACTOR_THREADPOOL_MAXSIZE` -- :setting:`TWISTED_REACTOR` +- :setting:`TWISTED_REACTOR` (ignored when using + :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below) :setting:`ASYNCIO_EVENT_LOOP` and :setting:`TWISTED_REACTOR` are used upon installing the reactor. The rest of the settings are applied when starting the reactor. +There is an additional restriction for :setting:`TWISTED_REACTOR` and +:setting:`ASYNCIO_EVENT_LOOP` when using +:class:`~scrapy.crawler.AsyncCrawlerProcess`: when this class is instantiated, +it installs :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, +ignoring the value of :setting:`TWISTED_REACTOR` and using the value of +:setting:`ASYNCIO_EVENT_LOOP` that was passed to +:meth:`AsyncCrawlerProcess.__init__() +`. If a different value for +:setting:`TWISTED_REACTOR` or :setting:`ASYNCIO_EVENT_LOOP` is provided later, +e.g. in :ref:`per-spider settings `, an exception will be +raised. + .. _topics-settings-ref: @@ -1977,9 +1991,11 @@ Import path of a given :mod:`~twisted.internet.reactor`. Scrapy will install this reactor if no other reactor is installed yet, such as when the ``scrapy`` CLI program is invoked or when using the +:class:`~scrapy.crawler.AsyncCrawlerProcess` class or the :class:`~scrapy.crawler.CrawlerProcess` class. -If you are using the :class:`~scrapy.crawler.CrawlerRunner` class, you also +If you are using the :class:`~scrapy.crawler.AsyncCrawlerRunner` class or the +:class:`~scrapy.crawler.CrawlerRunner` class, you also need to install the correct reactor manually. You can do that using :func:`~scrapy.utils.reactor.install_reactor`: @@ -1988,12 +2004,12 @@ need to install the correct reactor manually. You can do that using If a reactor is already installed, :func:`~scrapy.utils.reactor.install_reactor` has no effect. -:meth:`CrawlerRunner.__init__ ` raises -:exc:`Exception` if the installed reactor does not match the +:class:`~scrapy.crawler.AsyncCrawlerRunner` and other similar classes raise an +exception if the installed reactor does not match the :setting:`TWISTED_REACTOR` setting; therefore, having top-level :mod:`~twisted.internet.reactor` imports in project files and imported -third-party libraries will make Scrapy raise :exc:`Exception` when -it checks which reactor is installed. +third-party libraries will make Scrapy raise an exception when it checks which +reactor is installed. In order to use the reactor installed by Scrapy: @@ -2025,7 +2041,7 @@ In order to use the reactor installed by Scrapy: self.crawler.engine.close_spider(self, "timeout") -which raises :exc:`Exception`, becomes: +which raises an exception, becomes: .. code-block:: python diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b0d9a5452..721c81d81 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -22,6 +22,7 @@ from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest from scrapy.http import Request, Response from scrapy.utils.defer import ( deferred_f_from_coro_f, + deferred_from_coro, maybe_deferred_to_future, ) from scrapy.utils.log import failure_to_exc_info, logformatter_adapter @@ -122,8 +123,10 @@ class ExecutionEngine: ) return scheduler_cls - @deferred_f_from_coro_f - async def start(self, _start_request_processing=True) -> None: + def start(self, _start_request_processing=True) -> Deferred[None]: + return deferred_from_coro(self.start_async(_start_request_processing)) + + async def start_async(self, _start_request_processing=True) -> None: if self.running: raise RuntimeError("Engine already running") self.start_time = time() @@ -392,10 +395,15 @@ class ExecutionEngine: finally: self._slot.nextcall.schedule() - @deferred_f_from_coro_f - async def open_spider( + def open_spider(self, spider: Spider, close_if_idle: bool = True) -> Deferred[None]: + return deferred_from_coro( + self.open_spider_async(spider, close_if_idle=close_if_idle) + ) + + async def open_spider_async( self, spider: Spider, + *, close_if_idle: bool = True, ) -> None: if self._slot is not None: diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 1d6532fa9..c22b8603b 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -21,7 +21,8 @@ from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager -from scrapy.utils.defer import deferred_to_future +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.defer import deferred_from_coro, deferred_to_future from scrapy.utils.log import ( LogCounterHandler, configure_logging, @@ -33,8 +34,10 @@ from scrapy.utils.log import ( from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.reactor import ( + _asyncio_reactor_path, install_reactor, is_asyncio_reactor_installed, + is_reactor_installed, verify_installed_asyncio_event_loop, verify_installed_reactor, ) @@ -142,6 +145,12 @@ class Crawler: # this method. @inlineCallbacks def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]: + """Start the crawler by instantiating its spider class with the given + *args* and *kwargs* arguments, while setting the execution engine in + motion. Should be called only once. + + Return a deferred that is fired when the crawl is finished. + """ if self.crawling: raise RuntimeError("Crawling already taking place") if self._started: @@ -163,6 +172,42 @@ class Crawler: yield self.engine.close() raise + async def crawl_async(self, *args: Any, **kwargs: Any) -> None: + """Start the crawler by instantiating its spider class with the given + *args* and *kwargs* arguments, while setting the execution engine in + motion. Should be called only once. + + .. versionadded:: VERSION + + Complete when the crawl is finished. + + This function requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be + installed. + """ + if not is_asyncio_available(): + raise RuntimeError("Crawler.crawl_async() requires AsyncioSelectorReactor.") + if self.crawling: + raise RuntimeError("Crawling already taking place") + if self._started: + raise RuntimeError( + "Cannot run Crawler.crawl_async() more than once on the same instance." + ) + self.crawling = self._started = True + + try: + self.spider = self._create_spider(*args, **kwargs) + self._apply_settings() + self._update_root_log_handler() + self.engine = self._create_engine() + await self.engine.open_spider_async(self.spider) + await self.engine.start_async() + except Exception: + self.crawling = False + if self.engine is not None: + await deferred_to_future(self.engine.close()) + raise + def _create_spider(self, *args: Any, **kwargs: Any) -> Spider: return self.spidercls.from_crawler(self, *args, **kwargs) @@ -171,13 +216,26 @@ class Crawler: @inlineCallbacks def stop(self) -> Generator[Deferred[Any], Any, None]: - """Starts a graceful stop of the crawler and returns a deferred that is + """Start a graceful stop of the crawler and return a deferred that is fired when the crawler is stopped.""" if self.crawling: self.crawling = False assert self.engine yield self.engine.stop() + async def stop_async(self) -> None: + """Start a graceful stop of the crawler and complete when the crawler is stopped. + + .. versionadded:: VERSION + + This function requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be + installed. + """ + if not is_asyncio_available(): + raise RuntimeError("Crawler.stop_async() requires AsyncioSelectorReactor.") + await deferred_to_future(self.stop()) + @staticmethod def _get_component( component_class: type[_T], components: Iterable[Any] @@ -318,9 +376,6 @@ class CrawlerRunnerBase: spidercls = self.spider_loader.load(spidercls) return Crawler(spidercls, self.settings) - def _stop(self) -> Deferred[Any]: - return DeferredList([c.stop() for c in list(self.crawlers)]) - class CrawlerRunner(CrawlerRunnerBase): """ @@ -397,7 +452,7 @@ class CrawlerRunner(CrawlerRunnerBase): Returns a deferred that is fired when they all have ended. """ - return self._stop() + return DeferredList(c.stop() for c in self.crawlers) @inlineCallbacks def join(self) -> Generator[Deferred[Any], Any, None]: @@ -429,14 +484,14 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): def __init__(self, settings: dict[str, Any] | Settings | None = None): super().__init__(settings) - self._active: set[asyncio.Future[None]] = set() + self._active: set[asyncio.Task[None]] = set() def crawl( self, crawler_or_spidercls: type[Spider] | str | Crawler, *args: Any, **kwargs: Any, - ) -> asyncio.Future[None]: + ) -> asyncio.Task[None]: """ Run a crawler with the provided arguments. @@ -447,7 +502,7 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): instance, this method will try to create one using this parameter as the spider class given to it. - Returns a :class:`~asyncio.Future` object which completes when the + Returns a :class:`~asyncio.Task` object which completes when the crawling is finished. :param crawler_or_spidercls: already created crawler, or a spider class @@ -465,24 +520,27 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): "it must be a spider class (or a Crawler object)" ) if not is_asyncio_reactor_installed(): - raise RuntimeError("AsyncCrawlerRunner requires AsyncioSelectorReactor.") + raise RuntimeError( + f"{type(self).__name__} requires AsyncioSelectorReactor." + ) crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl( - self, crawler: Crawler, *args: Any, **kwargs: Any - ) -> asyncio.Future[None]: + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]: + # At this point the asyncio loop has been installed either by the user + # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()). + loop = asyncio.get_event_loop() self.crawlers.add(crawler) - future = deferred_to_future(crawler.crawl(*args, **kwargs)) - self._active.add(future) + task = loop.create_task(crawler.crawl_async(*args, **kwargs)) + self._active.add(task) - def _done(_: asyncio.Future[None]) -> None: + def _done(_: asyncio.Task[None]) -> None: self.crawlers.discard(crawler) - self._active.discard(future) + self._active.discard(task) self.bootstrap_failed |= not getattr(crawler, "spider", None) - future.add_done_callback(_done) - return future + task.add_done_callback(_done) + return task async def stop(self) -> None: """ @@ -490,7 +548,10 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): Completes when they all have ended. """ - await deferred_to_future(self._stop()) + if self.crawlers: + await asyncio.wait( + [asyncio.create_task(c.stop_async()) for c in self.crawlers] + ) async def join(self) -> None: """ @@ -498,33 +559,10 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): executions. """ while self._active: - await asyncio.gather(*self._active) + await asyncio.wait(self._active) -class CrawlerProcess(CrawlerRunner): - """ - A class to run multiple scrapy crawlers in a process simultaneously. - - This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support - for starting a :mod:`~twisted.internet.reactor` and handling shutdown - signals, like the keyboard interrupt command Ctrl-C. It also configures - top-level logging. - - This utility should be a better fit than - :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another - :mod:`~twisted.internet.reactor` within your application. - - The CrawlerProcess object must be instantiated with a - :class:`~scrapy.settings.Settings` object. - - :param install_root_handler: whether to install root logging handler - (default: True) - - This class shouldn't be needed (since Scrapy is responsible of using it - accordingly) unless writing scripts that manually handle the crawling - process. See :ref:`run-from-script` for an example. - """ - +class CrawlerProcessBase(CrawlerRunnerBase): def __init__( self, settings: dict[str, Any] | Settings | None = None, @@ -533,7 +571,6 @@ class CrawlerProcess(CrawlerRunner): super().__init__(settings) configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) - self._initialized_reactor: bool = False def _signal_shutdown(self, signum: int, _: Any) -> None: from twisted.internet import reactor @@ -556,6 +593,75 @@ class CrawlerProcess(CrawlerRunner): ) reactor.callFromThread(self._stop_reactor) + def _setup_reactor(self, install_signal_handlers: bool) -> None: + from twisted.internet import reactor + + resolver_class = load_object(self.settings["DNS_RESOLVER"]) + # We pass self, which is CrawlerProcess, instead of Crawler here, + # which works because the default resolvers only use crawler.settings. + resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type] + resolver.install_on_reactor() + tp = reactor.getThreadPool() + tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) + reactor.addSystemEventTrigger("before", "shutdown", self._stop_dfd) + if install_signal_handlers: + reactor.addSystemEventTrigger( + "after", "startup", install_shutdown_handlers, self._signal_shutdown + ) + + def _stop_dfd(self) -> Deferred[Any]: + raise NotImplementedError + + @inlineCallbacks + def _graceful_stop_reactor(self) -> Generator[Deferred[Any], Any, None]: + try: + yield self._stop_dfd() + finally: + self._stop_reactor() + + def _stop_reactor(self, _: Any = None) -> None: + from twisted.internet import reactor + + # raised if already stopped or in shutdown stage + with contextlib.suppress(RuntimeError): + reactor.stop() + + +class CrawlerProcess(CrawlerProcessBase, CrawlerRunner): + """ + A class to run multiple scrapy crawlers in a process simultaneously. + + This class extends :class:`~scrapy.crawler.CrawlerRunner` by adding support + for starting a :mod:`~twisted.internet.reactor` and handling shutdown + signals, like the keyboard interrupt command Ctrl-C. It also configures + top-level logging. + + This utility should be a better fit than + :class:`~scrapy.crawler.CrawlerRunner` if you aren't running another + :mod:`~twisted.internet.reactor` within your application. + + The CrawlerProcess object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + :param install_root_handler: whether to install root logging handler + (default: True) + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides Deferred-based APIs. Use :class:`AsyncCrawlerProcess` + for modern coroutine APIs. + """ + + def __init__( + self, + settings: dict[str, Any] | Settings | None = None, + install_root_handler: bool = True, + ): + super().__init__(settings, install_root_handler) + self._initialized_reactor: bool = False + def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler: if isinstance(spidercls, str): spidercls = self.spider_loader.load(spidercls) @@ -563,6 +669,9 @@ class CrawlerProcess(CrawlerRunner): self._initialized_reactor = True return Crawler(spidercls, self.settings, init_reactor=init_reactor) + def _stop_dfd(self) -> Deferred[Any]: + return self.stop() + def start( self, stop_after_crawl: bool = True, install_signal_handlers: bool = True ) -> None: @@ -589,30 +698,86 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) - resolver_class = load_object(self.settings["DNS_RESOLVER"]) - # We pass self, which is CrawlerProcess, instead of Crawler here, - # which works because the default resolvers only use crawler.settings. - resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[arg-type] - resolver.install_on_reactor() - tp = reactor.getThreadPool() - tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) - reactor.addSystemEventTrigger("before", "shutdown", self.stop) - if install_signal_handlers: - reactor.addSystemEventTrigger( - "after", "startup", install_shutdown_handlers, self._signal_shutdown - ) + self._setup_reactor(install_signal_handlers) reactor.run(installSignalHandlers=install_signal_handlers) # blocking call - @inlineCallbacks - def _graceful_stop_reactor(self) -> Generator[Deferred[Any], Any, None]: - try: - yield self.stop() - finally: - self._stop_reactor() - def _stop_reactor(self, _: Any = None) -> None: +class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): + """ + A class to run multiple scrapy crawlers in a process simultaneously. + + This class extends :class:`~scrapy.crawler.AsyncCrawlerRunner` by adding support + for starting a :mod:`~twisted.internet.reactor` and handling shutdown + signals, like the keyboard interrupt command Ctrl-C. It also configures + top-level logging. + + This utility should be a better fit than + :class:`~scrapy.crawler.AsyncCrawlerRunner` if you aren't running another + :mod:`~twisted.internet.reactor` within your application. + + The AsyncCrawlerProcess object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + :param install_root_handler: whether to install root logging handler + (default: True) + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides coroutine APIs. It requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + """ + + def __init__( + self, + settings: dict[str, Any] | Settings | None = None, + install_root_handler: bool = True, + ): + super().__init__(settings, install_root_handler) + # We want the asyncio event loop to be installed early, so that it's + # always the correct one. And as we do that, we can also install the + # reactor here. + # The ASYNCIO_EVENT_LOOP setting cannot be overridden by add-ons and + # spiders when using AsyncCrawlerProcess. + loop_path = self.settings["ASYNCIO_EVENT_LOOP"] + if is_reactor_installed(): + # The user could install a reactor before this class is instantiated. + # We need to make sure the reactor is the correct one and the loop + # type matches the setting. + verify_installed_reactor(_asyncio_reactor_path) + if loop_path: + verify_installed_asyncio_event_loop(loop_path) + else: + install_reactor(_asyncio_reactor_path, loop_path) + self._initialized_reactor = True + + def _stop_dfd(self) -> Deferred[Any]: + return deferred_from_coro(self.stop()) + + def start( + self, stop_after_crawl: bool = True, install_signal_handlers: bool = True + ) -> None: + """ + This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool + size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache + based on :setting:`DNSCACHE_ENABLED` and :setting:`DNSCACHE_SIZE`. + + If ``stop_after_crawl`` is True, the reactor will be stopped after all + crawlers have finished, using :meth:`join`. + + :param bool stop_after_crawl: stop or not the reactor when all + crawlers have finished + + :param bool install_signal_handlers: whether to install the OS signal + handlers from Twisted and Scrapy (default: True) + """ from twisted.internet import reactor - # raised if already stopped or in shutdown stage - with contextlib.suppress(RuntimeError): - reactor.stop() + if stop_after_crawl: + loop = asyncio.get_event_loop() + join_task = loop.create_task(self.join()) + join_task.add_done_callback(self._stop_reactor) + + self._setup_reactor(install_signal_handlers) + reactor.run(installSignalHandlers=install_signal_handlers) # blocking call diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 4649c4daa..2df82c8f2 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -23,7 +23,6 @@ from twisted.python import failure from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning from scrapy.utils.asyncio import is_asyncio_available -from scrapy.utils.reactor import _get_asyncio_event_loop if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable @@ -385,8 +384,7 @@ def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2: # that use asyncio, e.g. "await asyncio.sleep(1)" return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o)) # wrapping the coroutine into a Future and then into a Deferred, this requires AsyncioSelectorReactor - event_loop = _get_asyncio_event_loop() - return Deferred.fromFuture(asyncio.ensure_future(o, loop=event_loop)) + return Deferred.fromFuture(asyncio.ensure_future(o)) return o @@ -430,6 +428,10 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]: Return an :class:`asyncio.Future` object that wraps *d*. + This function requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be + installed. + When :ref:`using the asyncio reactor `, you cannot await on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy callables defined as coroutines `, you can only await on @@ -442,8 +444,15 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]: additional_request = scrapy.Request('https://example.org/price') deferred = self.crawler.engine.download(additional_request) additional_response = await deferred_to_future(deferred) + + .. versionchanged:: VERSION + This function no longer installs an asyncio loop if called before the + Twisted asyncio reactor is installed. A :exc:`RuntimeError` is raised + in this case. """ - return d.asFuture(_get_asyncio_event_loop()) + if not is_asyncio_available(): + raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.") + return d.asFuture(asyncio.get_event_loop()) def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 5e76da37b..2fb1e0ce7 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -10,6 +10,7 @@ from twisted.internet import asyncioreactor, error from twisted.internet.defer import Deferred from scrapy.utils.misc import load_object +from scrapy.utils.python import global_object_name if TYPE_CHECKING: from asyncio import AbstractEventLoop, AbstractEventLoopPolicy @@ -87,6 +88,9 @@ class CallLaterOnce(Generic[_T]): await maybe_deferred_to_future(d) +_asyncio_reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" + + def set_asyncio_event_loop_policy() -> None: """The policy functions from asyncio often behave unexpectedly, so we restrict their use to the absolutely essential case. @@ -161,21 +165,34 @@ def set_asyncio_event_loop(event_loop_path: str | None) -> AbstractEventLoop: def verify_installed_reactor(reactor_path: str) -> None: - """Raises :exc:`Exception` if the installed + """Raise :exc:`RuntimeError` if the installed :mod:`~twisted.internet.reactor` does not match the specified import - path.""" + path or if no reactor is installed.""" + if not is_reactor_installed(): + raise RuntimeError( + "verify_installed_reactor() called without an installed reactor." + ) + from twisted.internet import reactor - reactor_class = load_object(reactor_path) - if not reactor.__class__ == reactor_class: + expected_reactor_type = load_object(reactor_path) + reactor_type = type(reactor) + if not reactor_type == expected_reactor_type: raise RuntimeError( - "The installed reactor " - f"({reactor.__module__}.{reactor.__class__.__name__}) does not " - f"match the requested one ({reactor_path})" + f"The installed reactor ({global_object_name(reactor_type)}) " + f"does not match the requested one ({reactor_path})" ) def verify_installed_asyncio_event_loop(loop_path: str) -> None: + """Raise :exc:`RuntimeError` if the even loop of the installed + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` + does not match the specified import path or if no reactor is installed.""" + if not is_reactor_installed(): + raise RuntimeError( + "verify_installed_asyncio_event_loop() called without an installed reactor." + ) + from twisted.internet import reactor loop_class = load_object(loop_path) @@ -185,16 +202,16 @@ def verify_installed_asyncio_event_loop(loop_path: str) -> None: f"{reactor._asyncioEventloop.__class__.__module__}" f".{reactor._asyncioEventloop.__class__.__qualname__}" ) - specified = f"{loop_class.__module__}.{loop_class.__qualname__}" raise RuntimeError( "Scrapy found an asyncio Twisted reactor already " f"installed, and its event loop class ({installed}) does " "not match the one specified in the ASYNCIO_EVENT_LOOP " - f"setting ({specified})" + f"setting ({global_object_name(loop_class)})" ) def is_reactor_installed() -> bool: + """Check whether a :mod:`~twisted.internet.reactor` is installed.""" return "twisted.internet.reactor" in sys.modules diff --git a/tests/AsyncCrawlerProcess/args_settings.py b/tests/AsyncCrawlerProcess/args_settings.py new file mode 100644 index 000000000..5e162e782 --- /dev/null +++ b/tests/AsyncCrawlerProcess/args_settings.py @@ -0,0 +1,25 @@ +from typing import Any + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess, Crawler + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + @classmethod + def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any): + spider = super().from_crawler(crawler, *args, **kwargs) + spider.settings.set("FOO", kwargs.get("foo")) + return spider + + async def start(self): + self.logger.info(f"The value of FOO is {self.settings.getint('FOO')}") + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider, foo=42) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop.py new file mode 100644 index 000000000..172e36b7b --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop.py @@ -0,0 +1,20 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py new file mode 100644 index 000000000..d76da51a1 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_different.py @@ -0,0 +1,23 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + custom_settings = { + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": None, + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py new file mode 100644 index 000000000..bd4a99e14 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_custom_loop_custom_settings_same.py @@ -0,0 +1,23 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + custom_settings = { + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py new file mode 100644 index 000000000..c32aaf37d --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import asyncio +import sys + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.defer import deferred_from_coro + + +class UppercasePipeline: + async def _open_spider(self, spider): + spider.logger.info("async pipeline opened!") + await asyncio.sleep(0.1) + + def open_spider(self, spider): + return deferred_from_coro(self._open_spider(spider)) + + def process_item(self, item, spider): + return {"url": item["url"].upper()} + + +class UrlSpider(Spider): + name = "url_spider" + start_urls = ["data:,"] + custom_settings = { + "ITEM_PIPELINES": {UppercasePipeline: 100}, + } + + def parse(self, response): + yield {"url": response.url} + + +if __name__ == "__main__": + ASYNCIO_EVENT_LOOP: str | None + try: + ASYNCIO_EVENT_LOOP = sys.argv[1] + except IndexError: + ASYNCIO_EVENT_LOOP = None + + process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": ASYNCIO_EVENT_LOOP, + } + ) + process.crawl(UrlSpider) + process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py new file mode 100644 index 000000000..3c47eb826 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_no_reactor.py @@ -0,0 +1,27 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactor import is_asyncio_reactor_installed + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py new file mode 100644 index 000000000..e025e17d1 --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor.py @@ -0,0 +1,53 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + is_reactor_installed, +) + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed before is_asyncio_reactor_installed()." + ) + +try: + is_asyncio_reactor_installed() +except RuntimeError: + pass +else: + raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.") + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed after is_asyncio_reactor_installed()." + ) + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + +if not is_asyncio_reactor_installed(): + raise RuntimeError("Wrong reactor installed after install_reactor().") + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py new file mode 100644 index 000000000..4257bc0ac --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -0,0 +1,29 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncioreactor.install(asyncio.get_event_loop()) + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py new file mode 100644 index 000000000..9c6fd090b --- /dev/null +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -0,0 +1,31 @@ +import asyncio +import sys + +from twisted.internet import asyncioreactor +from uvloop import Loop + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if sys.platform == "win32": + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) +asyncio.set_event_loop(Loop()) +asyncioreactor.install(asyncio.get_event_loop()) + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } +) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py new file mode 100644 index 000000000..5f75d5e17 --- /dev/null +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py @@ -0,0 +1,35 @@ +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes in a finite amount of time (does not hang indefinitely in the DNS resolution) + """ + + name = "caching_hostname_resolver_spider" + + async def start(self): + yield scrapy.Request(self.url) + + def parse(self, response): + for _ in range(10): + yield scrapy.Request( + response.url, dont_filter=True, callback=self.ignore_response + ) + + def ignore_response(self, response): + self.logger.info(repr(response.ip_address)) + + +if __name__ == "__main__": + process = AsyncCrawlerProcess( + settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + } + ) + process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) + process.start() diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py new file mode 100644 index 000000000..c43f0a9c2 --- /dev/null +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class CachingHostnameResolverSpider(scrapy.Spider): + """ + Finishes without a twisted.internet.error.DNSLookupError exception + """ + + name = "caching_hostname_resolver_spider" + start_urls = ["http://[::1]"] + + +if __name__ == "__main__": + process = AsyncCrawlerProcess( + settings={ + "RETRY_ENABLED": False, + "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + } + ) + process.crawl(CachingHostnameResolverSpider) + process.start() diff --git a/tests/AsyncCrawlerProcess/default_name_resolver.py b/tests/AsyncCrawlerProcess/default_name_resolver.py new file mode 100644 index 000000000..af56ccd01 --- /dev/null +++ b/tests/AsyncCrawlerProcess/default_name_resolver.py @@ -0,0 +1,18 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class IPv6Spider(scrapy.Spider): + """ + Raises a twisted.internet.error.DNSLookupError: + the default name resolver does not handle IPv6 addresses. + """ + + name = "ipv6_spider" + start_urls = ["http://[::1]"] + + +if __name__ == "__main__": + process = AsyncCrawlerProcess(settings={"RETRY_ENABLED": False}) + process.crawl(IPv6Spider) + process.start() diff --git a/tests/AsyncCrawlerProcess/multi.py b/tests/AsyncCrawlerProcess/multi.py new file mode 100644 index 000000000..2eede5471 --- /dev/null +++ b/tests/AsyncCrawlerProcess/multi.py @@ -0,0 +1,17 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/reactor_default.py b/tests/AsyncCrawlerProcess/reactor_default.py new file mode 100644 index 000000000..9638652bd --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactor_default.py @@ -0,0 +1,18 @@ +from twisted.internet import reactor # noqa: F401,TID253 + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +d = process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/simple.py b/tests/AsyncCrawlerProcess/simple.py new file mode 100644 index 000000000..d24b4f193 --- /dev/null +++ b/tests/AsyncCrawlerProcess/simple.py @@ -0,0 +1,16 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class NoRequestsSpider(scrapy.Spider): + name = "no_request" + + async def start(self): + return + yield + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(NoRequestsSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/sleeping.py b/tests/AsyncCrawlerProcess/sleeping.py new file mode 100644 index 000000000..88caf5032 --- /dev/null +++ b/tests/AsyncCrawlerProcess/sleeping.py @@ -0,0 +1,20 @@ +import asyncio +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class SleepingSpider(scrapy.Spider): + name = "sleeping" + + start_urls = ["data:,;"] + + async def parse(self, response): + await asyncio.sleep(int(sys.argv[1])) + + +process = AsyncCrawlerProcess(settings={}) + +process.crawl(SleepingSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py b/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py new file mode 100644 index 000000000..dc820ea3a --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_asyncio.py @@ -0,0 +1,15 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = "asyncio_reactor" + + +process = AsyncCrawlerProcess( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } +) +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py new file mode 100644 index 000000000..5fd48274a --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings.py @@ -0,0 +1,14 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class AsyncioReactorSpider(scrapy.Spider): + name = "asyncio_reactor" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = AsyncCrawlerProcess() +process.crawl(AsyncioReactorSpider) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py new file mode 100644 index 000000000..c205c3cd2 --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_same.py @@ -0,0 +1,22 @@ +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + + +class AsyncioReactorSpider1(scrapy.Spider): + name = "asyncio_reactor1" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +class AsyncioReactorSpider2(scrapy.Spider): + name = "asyncio_reactor2" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + + +process = AsyncCrawlerProcess() +process.crawl(AsyncioReactorSpider1) +process.crawl(AsyncioReactorSpider2) +process.start() diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py new file mode 100644 index 000000000..68239e651 --- /dev/null +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess + +if TYPE_CHECKING: + from asyncio import Task + + +class AsyncioReactorSpider(scrapy.Spider): + name = "asyncio_reactor" + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor", + } + + +def log_task_exception(task: Task) -> None: + try: + task.result() + except Exception: + logging.exception("Crawl task failed") + + +process = AsyncCrawlerProcess() +task = process.crawl(AsyncioReactorSpider) +task.add_done_callback(log_task_exception) +process.start() diff --git a/tests/AsyncCrawlerRunner/custom_loop_different.py b/tests/AsyncCrawlerRunner/custom_loop_different.py new file mode 100644 index 000000000..89cf0e536 --- /dev/null +++ b/tests/AsyncCrawlerRunner/custom_loop_different.py @@ -0,0 +1,31 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/custom_loop_same.py b/tests/AsyncCrawlerRunner/custom_loop_same.py new file mode 100644 index 000000000..43d0dc053 --- /dev/null +++ b/tests/AsyncCrawlerRunner/custom_loop_same.py @@ -0,0 +1,31 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor", "uvloop.Loop") +react(main) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py index d8c467f40..7c50277b9 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -4,12 +4,12 @@ import sys from twisted.internet import asyncioreactor from twisted.python import log +import scrapy +from scrapy.crawler import CrawlerProcess + if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncioreactor.install(asyncio.get_event_loop()) - -import scrapy # noqa: E402 -from scrapy.crawler import CrawlerProcess # noqa: E402 +asyncioreactor.install() class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py index e7d3ca9cc..578e0029d 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -4,13 +4,13 @@ import sys from twisted.internet import asyncioreactor from uvloop import Loop +import scrapy +from scrapy.crawler import CrawlerProcess + if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.set_event_loop(Loop()) -asyncioreactor.install(asyncio.get_event_loop()) - -import scrapy # noqa: E402 -from scrapy.crawler import CrawlerProcess # noqa: E402 +asyncioreactor.install() class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerRunner/custom_loop_different.py b/tests/CrawlerRunner/custom_loop_different.py new file mode 100644 index 000000000..86ba1ed47 --- /dev/null +++ b/tests/CrawlerRunner/custom_loop_different.py @@ -0,0 +1,29 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/CrawlerRunner/custom_loop_same.py b/tests/CrawlerRunner/custom_loop_same.py new file mode 100644 index 000000000..98b8dde87 --- /dev/null +++ b/tests/CrawlerRunner/custom_loop_same.py @@ -0,0 +1,29 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "ASYNCIO_EVENT_LOOP": "uvloop.Loop", + } + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor", "uvloop.Loop") +react(main) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index a1d3c02fb..ce5963cd7 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,3 +1,4 @@ +import asyncio import logging import platform import re @@ -5,6 +6,7 @@ import signal import subprocess import sys import warnings +from abc import ABC, abstractmethod from pathlib import Path from typing import Any @@ -18,12 +20,18 @@ from zope.interface.exceptions import MultipleInvalid import scrapy from scrapy import Spider -from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerProcess, CrawlerRunner +from scrapy.crawler import ( + AsyncCrawlerProcess, + AsyncCrawlerRunner, + Crawler, + CrawlerProcess, + CrawlerRunner, +) from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader -from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.defer import deferred_f_from_coro_f, deferred_from_coro from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings @@ -88,12 +96,39 @@ class TestCrawler(TestBaseCrawler): Crawler(DefaultSpider()) @inlineCallbacks - def test_crawler_crawl_twice_unsupported(self): + def test_crawler_crawl_twice_seq_unsupported(self): crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) yield crawler.crawl() with pytest.raises(RuntimeError, match="more than once on the same instance"): yield crawler.crawl() + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_crawler_crawl_async_twice_seq_unsupported(self): + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + await crawler.crawl_async() + with pytest.raises(RuntimeError, match="more than once on the same instance"): + await crawler.crawl_async() + + @inlineCallbacks + def test_crawler_crawl_twice_parallel_unsupported(self): + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + d1 = crawler.crawl() + d2 = crawler.crawl() + yield d1 + with pytest.raises(RuntimeError, match="Crawling already taking place"): + yield d2 + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_crawler_crawl_async_twice_parallel_unsupported(self): + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + t1 = asyncio.create_task(crawler.crawl_async()) + t2 = asyncio.create_task(crawler.crawl_async()) + await t1 + with pytest.raises(RuntimeError, match="Crawling already taking place"): + await t2 + def test_get_addon(self): class ParentAddon: pass @@ -590,6 +625,18 @@ class TestCrawlerProcess(TestBaseCrawler): self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") +@pytest.mark.only_asyncio +class TestAsyncCrawlerProcess(TestBaseCrawler): + def test_crawler_process_accepts_dict(self): + runner = AsyncCrawlerProcess({"foo": "bar"}) + assert runner.settings["foo"] == "bar" + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + def test_crawler_process_accepts_None(self): + runner = AsyncCrawlerProcess() + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + class ExceptionSpider(scrapy.Spider): name = "exception" @@ -692,8 +739,15 @@ class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): pytest.skip("This test is only for CrawlerRunner") -class ScriptRunnerMixin: - script_dir: Path +class ScriptRunnerMixin(ABC): + @property + @abstractmethod + def script_dir(self) -> Path: + raise NotImplementedError + + @staticmethod + def get_script_dir(name: str) -> Path: + return Path(__file__).parent.resolve() / name def get_script_args(self, script_name: str, *script_args: str) -> list[str]: script_path = self.script_dir / script_name @@ -711,8 +765,10 @@ class ScriptRunnerMixin: return stderr.decode("utf-8") -class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): - script_dir = Path(__file__).parent.resolve() / "CrawlerProcess" +class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin, unittest.TestCase): + """Common tests between CrawlerProcess and AsyncCrawlerProcess, + with the same file names and expectations. + """ def test_simple(self): log = self.run_script("simple.py") @@ -739,48 +795,6 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" ) in log - def test_reactor_default_twisted_reactor_select(self): - log = self.run_script("reactor_default_twisted_reactor_select.py") - if platform.system() in ["Windows", "Darwin"]: - # The goal of this test function is to test that, when a reactor is - # installed (the default one here) and a different reactor is - # configured (select here), an error raises. - # - # In Windows the default reactor is the select reactor, so that - # error does not raise. - # - # If that ever becomes the case on more platforms (i.e. if Linux - # also starts using the select reactor by default in a future - # version of Twisted), then we will need to rethink this test. - assert "Spider closed (finished)" in log - else: - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ) in log - - def test_reactor_select(self): - log = self.run_script("reactor_select.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" - ) in log - - def test_reactor_select_twisted_reactor_select(self): - log = self.run_script("reactor_select_twisted_reactor_select.py") - assert "Spider closed (finished)" in log - assert "ReactorAlreadyInstalledError" not in log - - def test_reactor_select_subclass_twisted_reactor_select(self): - log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") - assert "Spider closed (finished)" not in log - assert ( - "does not match the requested one " - "(twisted.internet.selectreactor.SelectReactor)" - ) in log - def test_asyncio_enabled_no_reactor(self): log = self.run_script("asyncio_enabled_no_reactor.py") assert "Spider closed (finished)" in log @@ -829,19 +843,6 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): assert "TimeoutError" not in log assert "twisted.internet.error.DNSLookupError" not in log - def test_twisted_reactor_select(self): - log = self.run_script("twisted_reactor_select.py") - assert "Spider closed (finished)" in log - assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log - - @pytest.mark.skipif( - platform.system() == "Windows", reason="PollReactor is not supported on Windows" - ) - def test_twisted_reactor_poll(self): - log = self.run_script("twisted_reactor_poll.py") - assert "Spider closed (finished)" in log - assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log - def test_twisted_reactor_asyncio(self): log = self.run_script("twisted_reactor_asyncio.py") assert "Spider closed (finished)" in log @@ -866,14 +867,6 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): in log ) - def test_twisted_reactor_asyncio_custom_settings_conflict(self): - log = self.run_script("twisted_reactor_custom_settings_conflict.py") - assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log - assert ( - "(twisted.internet.selectreactor.SelectReactor) does not match the requested one" - in log - ) - @pytest.mark.requires_uvloop def test_custom_loop_asyncio(self): log = self.run_script("asyncio_custom_loop.py") @@ -960,8 +953,113 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): p.wait() -class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): - script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" +class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("CrawlerProcess") + + def test_reactor_default_twisted_reactor_select(self): + log = self.run_script("reactor_default_twisted_reactor_select.py") + if platform.system() in ["Windows", "Darwin"]: + # The goal of this test function is to test that, when a reactor is + # installed (the default one here) and a different reactor is + # configured (select here), an error raises. + # + # In Windows the default reactor is the select reactor, so that + # error does not raise. + # + # If that ever becomes the case on more platforms (i.e. if Linux + # also starts using the select reactor by default in a future + # version of Twisted), then we will need to rethink this test. + assert "Spider closed (finished)" in log + else: + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + def test_reactor_select(self): + log = self.run_script("reactor_select.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log + + def test_reactor_select_twisted_reactor_select(self): + log = self.run_script("reactor_select_twisted_reactor_select.py") + assert "Spider closed (finished)" in log + assert "ReactorAlreadyInstalledError" not in log + + def test_reactor_select_subclass_twisted_reactor_select(self): + log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + def test_twisted_reactor_select(self): + log = self.run_script("twisted_reactor_select.py") + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + + @pytest.mark.skipif( + platform.system() == "Windows", reason="PollReactor is not supported on Windows" + ) + def test_twisted_reactor_poll(self): + log = self.run_script("twisted_reactor_poll.py") + assert "Spider closed (finished)" in log + assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log + + def test_twisted_reactor_asyncio_custom_settings_conflict(self): + log = self.run_script("twisted_reactor_custom_settings_conflict.py") + assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log + assert ( + "(twisted.internet.selectreactor.SelectReactor) does not match the requested one" + in log + ) + + +class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("AsyncCrawlerProcess") + + def test_twisted_reactor_custom_settings_select(self): + log = self.run_script("twisted_reactor_custom_settings_select.py") + assert "Spider closed (finished)" not in log + assert ( + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor) " + "does not match the requested one " + "(twisted.internet.selectreactor.SelectReactor)" + ) in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_same_loop(self): + log = self.run_script("asyncio_custom_loop_custom_settings_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_asyncio_enabled_reactor_different_loop(self): + log = self.run_script("asyncio_custom_loop_custom_settings_different.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + +class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): + """Common tests between CrawlerRunner and AsyncCrawlerRunner, + with the same file names and expectations. + """ def test_simple(self): log = self.run_script("simple.py") @@ -971,14 +1069,6 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): in log ) - def test_explicit_default_reactor(self): - log = self.run_script("explicit_default_reactor.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log - ) - def test_multi_parallel(self): log = self.run_script("multi_parallel.py") assert "Spider closed (finished)" in log @@ -1005,6 +1095,39 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): re.DOTALL, ) + @pytest.mark.requires_uvloop + def test_custom_loop_same(self): + log = self.run_script("custom_loop_same.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert "Using asyncio event loop: uvloop.Loop" in log + + @pytest.mark.requires_uvloop + def test_custom_loop_different(self): + log = self.run_script("custom_loop_different.py") + assert "Spider closed (finished)" not in log + assert ( + "does not match the one specified in the ASYNCIO_EVENT_LOOP " + "setting (uvloop.Loop)" + ) in log + + +class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("CrawlerRunner") + + def test_explicit_default_reactor(self): + log = self.run_script("explicit_default_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + def test_response_ip_address(self): log = self.run_script("ip_address.py") assert "INFO: Spider closed (finished)" in log @@ -1021,48 +1144,16 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): assert "DEBUG: Using asyncio event loop" in log -class TestAsyncCrawlerRunnerSubprocess(ScriptRunnerMixin): - script_dir = Path(__file__).parent.resolve() / "AsyncCrawlerRunner" - - def test_simple(self): - log = self.run_script("simple.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) +class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): + @property + def script_dir(self) -> Path: + return self.get_script_dir("AsyncCrawlerRunner") def test_simple_default_reactor(self): log = self.run_script("simple_default_reactor.py") assert "Spider closed (finished)" not in log assert "RuntimeError: AsyncCrawlerRunner requires AsyncioSelectorReactor" in log - def test_multi_parallel(self): - log = self.run_script("multi_parallel.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert re.search( - r"Spider opened.+Spider opened.+Closing spider.+Closing spider", - log, - re.DOTALL, - ) - - def test_multi_seq(self): - log = self.run_script("multi_seq.py") - assert "Spider closed (finished)" in log - assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - in log - ) - assert re.search( - r"Spider opened.+Closing spider.+Spider opened.+Closing spider", - log, - re.DOTALL, - ) - @pytest.mark.parametrize( ("settings", "items"), diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py index 99f175c60..eb00ab193 100644 --- a/tests/test_utils_reactor.py +++ b/tests/test_utils_reactor.py @@ -6,6 +6,7 @@ from twisted.trial.unittest import TestCase from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( + _asyncio_reactor_path, install_reactor, is_asyncio_reactor_installed, set_asyncio_event_loop, @@ -22,7 +23,7 @@ class TestAsyncio(TestCase): from twisted.internet import reactor as original_reactor with warnings.catch_warnings(record=True) as w: - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + install_reactor(_asyncio_reactor_path) assert len(w) == 0, [str(warning) for warning in w] from twisted.internet import reactor # pylint: disable=reimported @@ -31,5 +32,5 @@ class TestAsyncio(TestCase): @pytest.mark.only_asyncio @deferred_f_from_coro_f async def test_set_asyncio_event_loop(self): - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + install_reactor(_asyncio_reactor_path) assert set_asyncio_event_loop(None) is asyncio.get_running_loop()