diff --git a/conftest.py b/conftest.py index 8e0c429a0..18132b7e6 100644 --- a/conftest.py +++ b/conftest.py @@ -19,6 +19,8 @@ collect_ignore = [ "tests/mockserver.py", "tests/pipelines.py", "tests/spiders.py", + # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess + *_py_files("tests/AsyncCrawlerRunner"), # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 8e8f3a0c9..3e7bc45c5 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -109,6 +109,9 @@ how you :ref:`configure the downloader middlewares .. automethod:: stop +.. autoclass:: AsyncCrawlerRunner + :members: + .. autoclass:: CrawlerRunner :members: diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index db91cd073..18005aaf2 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -73,72 +73,87 @@ project as example. process.start() # the script will block here until the crawling is finished There's another Scrapy utility that provides more control over the crawling -process: :class:`scrapy.crawler.CrawlerRunner`. This class is a thin wrapper -that encapsulates some simple helpers to run multiple crawlers, but it won't -start or interfere with existing reactors in any way. +process: :class:`scrapy.crawler.AsyncCrawlerRunner` and +:class:`scrapy.crawler.CrawlerRunner`. These classes are thin wrappers +that encapsulate some simple helpers to run multiple crawlers, but they won't +start or interfere with existing reactors in any way. They have similar +functionality, differing in their asynchronous API style: +:class:`~scrapy.crawler.AsyncCrawlerRunner` returns coroutines from its +asynchronous methods while :class:`~scrapy.crawler.CrawlerRunner` returns +:class:`~twisted.internet.defer.Deferred` objects. -Using this class the reactor should be explicitly run after scheduling your -spiders. It's recommended you use :class:`~scrapy.crawler.CrawlerRunner` -instead of :class:`~scrapy.crawler.CrawlerProcess` if your application is -already using Twisted and you want to run Scrapy in the same reactor. +When using these classes the reactor should be explicitly run after scheduling +your spiders. It's recommended that you use +:class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.CrawlerRunner` instead of +:class:`~scrapy.crawler.CrawlerProcess` if your application is already using +Twisted and you want to run Scrapy in the same reactor. -Note that you will also have to shutdown the Twisted reactor yourself after the -spider is finished. This can be achieved by adding callbacks to the deferred -returned by the :meth:`CrawlerRunner.crawl -` method. +If you want to stop the reactor or run any other code right after the spider +finishes you can do that after the :meth:`AsyncCrawlerRunner.crawl() +` coroutine completes (or the Deferred +returned from :meth:`CrawlerRunner.crawl() +` fires). In the simplest case you can also +use :func:`twisted.internet.task.react` to start and stop the reactor, though +it may be easier to just use :class:`~scrapy.crawler.CrawlerProcess` instead. -Here's an example of its usage, along with a callback to manually stop the -reactor after ``MySpider`` has finished running. +Here's an example of using :class:`~scrapy.crawler.AsyncCrawlerRunner` together +with simple reactor management code: .. code-block:: python import scrapy - from scrapy.crawler import CrawlerRunner + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.log import configure_logging - - - class MySpider(scrapy.Spider): - # Your spider definition - ... - - - configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) - runner = CrawlerRunner() - - d = runner.crawl(MySpider) - - from twisted.internet import reactor - - d.addBoth(lambda _: reactor.stop()) - reactor.run() # the script will block here until the crawling is finished - -Same example but using a non-default reactor, it's only necessary call -``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically. - -.. code-block:: python - - import scrapy - from scrapy.crawler import CrawlerRunner - from scrapy.utils.log import configure_logging - - - class MySpider(scrapy.Spider): - # Your spider definition - ... - - - configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) - from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react + + + class MySpider(scrapy.Spider): + # Your spider definition + ... + + + async def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner() + await runner.crawl(MySpider) # completes when the spider finishes + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - runner = CrawlerRunner() - d = runner.crawl(MySpider) + react(deferred_f_from_coro_f(crawl)) - from twisted.internet import reactor +Same example but using :class:`~scrapy.crawler.CrawlerRunner` and a +different reactor (:class:`~scrapy.crawler.AsyncCrawlerRunner` only works +with :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`): - d.addBoth(lambda _: reactor.stop()) - reactor.run() # the script will block here until the crawling is finished +.. code-block:: python + + import scrapy + from scrapy.crawler import CrawlerRunner + from scrapy.utils.log import configure_logging + from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react + + + class MySpider(scrapy.Spider): + custom_settings = { + "TWISTED_REACTOR": "twisted.internet.epollreactor.EPollReactor", + } + # Your spider definition + ... + + + def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = CrawlerRunner() + d = runner.crawl(MySpider) + return d # this Deferred fires when the spider finishes + + + install_reactor("twisted.internet.epollreactor.EPollReactor") + react(crawl) .. seealso:: :doc:`twisted:core/howto/reactor-basics` @@ -176,14 +191,16 @@ Here is an example that runs multiple spiders simultaneously: process.crawl(MySpider2) process.start() # the script will block here until all crawling jobs are finished -Same example using :class:`~scrapy.crawler.CrawlerRunner`: +Same example using :class:`~scrapy.crawler.AsyncCrawlerRunner`: .. code-block:: python import scrapy - from scrapy.crawler import CrawlerRunner + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.log import configure_logging - from scrapy.utils.project import get_project_settings + from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react class MySpider1(scrapy.Spider): @@ -196,27 +213,29 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`: ... - configure_logging() - settings = get_project_settings() - runner = CrawlerRunner(settings) - runner.crawl(MySpider1) - runner.crawl(MySpider2) - d = runner.join() + async def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner() + runner.crawl(MySpider1) + runner.crawl(MySpider2) + await runner.join() # completes when both spiders finish - from twisted.internet import reactor - d.addBoth(lambda _: reactor.stop()) + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + react(deferred_f_from_coro_f(crawl)) - reactor.run() # the script will block here until all crawling jobs are finished -Same example but running the spiders sequentially by chaining the deferreds: +Same example but running the spiders sequentially by awaiting until each one +finishes before starting the next one: .. code-block:: python - from twisted.internet import defer - from scrapy.crawler import CrawlerRunner + import scrapy + from scrapy.crawler import AsyncCrawlerRunner + from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.log import configure_logging - from scrapy.utils.project import get_project_settings + from scrapy.utils.reactor import install_reactor + from twisted.internet.task import react class MySpider1(scrapy.Spider): @@ -229,22 +248,15 @@ Same example but running the spiders sequentially by chaining the deferreds: ... - settings = get_project_settings() - configure_logging(settings) - runner = CrawlerRunner(settings) + async def crawl(_): + configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"}) + runner = AsyncCrawlerRunner() + await runner.crawl(MySpider1) + await runner.crawl(MySpider2) - @defer.inlineCallbacks - def crawl(): - yield runner.crawl(MySpider1) - yield runner.crawl(MySpider2) - reactor.stop() - - - from twisted.internet import reactor - - crawl() - reactor.run() # the script will block here until the last crawl call is finished + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + react(deferred_f_from_coro_f(crawl)) .. note:: When running multiple spiders in the same process, :ref:`reactor settings ` should not have a different value per spider. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 5dbee6537..1d6532fa9 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import contextlib import logging import pprint @@ -20,6 +21,7 @@ from scrapy.extension import ExtensionManager from scrapy.interfaces import ISpiderLoader from scrapy.settings import BaseSettings, Settings, overridden_settings from scrapy.signalmanager import SignalManager +from scrapy.utils.defer import deferred_to_future from scrapy.utils.log import ( LogCounterHandler, configure_logging, @@ -263,19 +265,7 @@ class Crawler: return self._get_component(cls, self.engine.scraper.spidermw.middlewares) -class CrawlerRunner: - """ - This is a convenient helper class that keeps track of, manages and runs - crawlers inside an already setup :mod:`~twisted.internet.reactor`. - - The CrawlerRunner object must be instantiated with a - :class:`~scrapy.settings.Settings` object. - - This class shouldn't be needed (since Scrapy is responsible of using it - accordingly) unless writing scripts that manually handle the crawling - process. See :ref:`run-from-script` for an example. - """ - +class CrawlerRunnerBase: @staticmethod def _get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol: """Get SpiderLoader instance from settings""" @@ -293,7 +283,6 @@ class CrawlerRunner: self.settings: Settings = settings self.spider_loader: SpiderLoaderProtocol = self._get_spider_loader(settings) self._crawlers: set[Crawler] = set() - self._active: set[Deferred[None]] = set() self.bootstrap_failed = False @property @@ -302,6 +291,57 @@ class CrawlerRunner: :meth:`crawl` and managed by this class.""" return self._crawlers + def create_crawler( + self, crawler_or_spidercls: type[Spider] | str | Crawler + ) -> Crawler: + """ + Return a :class:`~scrapy.crawler.Crawler` object. + + * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. + * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler + is constructed for it. + * If ``crawler_or_spidercls`` is a string, this function finds + a spider with this name in a Scrapy project (using spider loader), + then creates a Crawler instance for it. + """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + "The crawler_or_spidercls argument cannot be a spider object, " + "it must be a spider class (or a Crawler object)" + ) + if isinstance(crawler_or_spidercls, Crawler): + return crawler_or_spidercls + return self._create_crawler(crawler_or_spidercls) + + def _create_crawler(self, spidercls: str | type[Spider]) -> Crawler: + if isinstance(spidercls, str): + spidercls = self.spider_loader.load(spidercls) + return Crawler(spidercls, self.settings) + + def _stop(self) -> Deferred[Any]: + return DeferredList([c.stop() for c in list(self.crawlers)]) + + +class CrawlerRunner(CrawlerRunnerBase): + """ + This is a convenient helper class that keeps track of, manages and runs + crawlers inside an already setup :mod:`~twisted.internet.reactor`. + + The CrawlerRunner object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides Deferred-based APIs. Use :class:`AsyncCrawlerRunner` + for modern coroutine APIs. + """ + + def __init__(self, settings: dict[str, Any] | Settings | None = None): + super().__init__(settings) + self._active: set[Deferred[None]] = set() + def crawl( self, crawler_or_spidercls: type[Spider] | str | Crawler, @@ -351,40 +391,13 @@ class CrawlerRunner: self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) - def create_crawler( - self, crawler_or_spidercls: type[Spider] | str | Crawler - ) -> Crawler: - """ - Return a :class:`~scrapy.crawler.Crawler` object. - - * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is. - * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler - is constructed for it. - * If ``crawler_or_spidercls`` is a string, this function finds - a spider with this name in a Scrapy project (using spider loader), - then creates a Crawler instance for it. - """ - if isinstance(crawler_or_spidercls, Spider): - raise ValueError( - "The crawler_or_spidercls argument cannot be a spider object, " - "it must be a spider class (or a Crawler object)" - ) - if isinstance(crawler_or_spidercls, Crawler): - return crawler_or_spidercls - return self._create_crawler(crawler_or_spidercls) - - def _create_crawler(self, spidercls: str | type[Spider]) -> Crawler: - if isinstance(spidercls, str): - spidercls = self.spider_loader.load(spidercls) - return Crawler(spidercls, self.settings) - def stop(self) -> Deferred[Any]: """ Stops simultaneously all the crawling jobs taking place. Returns a deferred that is fired when they all have ended. """ - return DeferredList([c.stop() for c in list(self.crawlers)]) + return self._stop() @inlineCallbacks def join(self) -> Generator[Deferred[Any], Any, None]: @@ -398,6 +411,96 @@ class CrawlerRunner: yield DeferredList(self._active) +class AsyncCrawlerRunner(CrawlerRunnerBase): + """ + This is a convenient helper class that keeps track of, manages and runs + crawlers inside an already setup :mod:`~twisted.internet.reactor`. + + The AsyncCrawlerRunner object must be instantiated with a + :class:`~scrapy.settings.Settings` object. + + This class shouldn't be needed (since Scrapy is responsible of using it + accordingly) unless writing scripts that manually handle the crawling + process. See :ref:`run-from-script` for an example. + + This class provides coroutine APIs. It requires + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + """ + + def __init__(self, settings: dict[str, Any] | Settings | None = None): + super().__init__(settings) + self._active: set[asyncio.Future[None]] = set() + + def crawl( + self, + crawler_or_spidercls: type[Spider] | str | Crawler, + *args: Any, + **kwargs: Any, + ) -> asyncio.Future[None]: + """ + Run a crawler with the provided arguments. + + It will call the given Crawler's :meth:`~Crawler.crawl` method, while + keeping track of it so it can be stopped later. + + If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler` + instance, this method will try to create one using this parameter as + the spider class given to it. + + Returns a :class:`~asyncio.Future` object which completes when the + crawling is finished. + + :param crawler_or_spidercls: already created crawler, or a spider class + or spider's name inside the project to create it + :type crawler_or_spidercls: :class:`~scrapy.crawler.Crawler` instance, + :class:`~scrapy.spiders.Spider` subclass or string + + :param args: arguments to initialize the spider + + :param kwargs: keyword arguments to initialize the spider + """ + if isinstance(crawler_or_spidercls, Spider): + raise ValueError( + "The crawler_or_spidercls argument cannot be a spider object, " + "it must be a spider class (or a Crawler object)" + ) + if not is_asyncio_reactor_installed(): + raise RuntimeError("AsyncCrawlerRunner requires AsyncioSelectorReactor.") + crawler = self.create_crawler(crawler_or_spidercls) + return self._crawl(crawler, *args, **kwargs) + + def _crawl( + self, crawler: Crawler, *args: Any, **kwargs: Any + ) -> asyncio.Future[None]: + self.crawlers.add(crawler) + future = deferred_to_future(crawler.crawl(*args, **kwargs)) + self._active.add(future) + + def _done(_: asyncio.Future[None]) -> None: + self.crawlers.discard(crawler) + self._active.discard(future) + self.bootstrap_failed |= not getattr(crawler, "spider", None) + + future.add_done_callback(_done) + return future + + async def stop(self) -> None: + """ + Stops simultaneously all the crawling jobs taking place. + + Completes when they all have ended. + """ + await deferred_to_future(self._stop()) + + async def join(self) -> None: + """ + Completes when all managed :attr:`crawlers` have completed their + executions. + """ + while self._active: + await asyncio.gather(*self._active) + + class CrawlerProcess(CrawlerRunner): """ A class to run multiple scrapy crawlers in a process simultaneously. @@ -458,7 +561,6 @@ class CrawlerProcess(CrawlerRunner): spidercls = self.spider_loader.load(spidercls) init_reactor = not self._initialized_reactor self._initialized_reactor = True - # temporary cast until self.spider_loader is typed return Crawler(spidercls, self.settings, init_reactor=init_reactor) def start( diff --git a/tests/AsyncCrawlerRunner/multi_parallel.py b/tests/AsyncCrawlerRunner/multi_parallel.py new file mode 100644 index 000000000..f1af9f794 --- /dev/null +++ b/tests/AsyncCrawlerRunner/multi_parallel.py @@ -0,0 +1,28 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + runner.crawl(NoRequestsSpider) + runner.crawl(NoRequestsSpider) + await runner.join() + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/multi_seq.py b/tests/AsyncCrawlerRunner/multi_seq.py new file mode 100644 index 000000000..987f7a514 --- /dev/null +++ b/tests/AsyncCrawlerRunner/multi_seq.py @@ -0,0 +1,27 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/simple.py b/tests/AsyncCrawlerRunner/simple.py new file mode 100644 index 000000000..140777b4f --- /dev/null +++ b/tests/AsyncCrawlerRunner/simple.py @@ -0,0 +1,26 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/AsyncCrawlerRunner/simple_default_reactor.py b/tests/AsyncCrawlerRunner/simple_default_reactor.py new file mode 100644 index 000000000..ae052f188 --- /dev/null +++ b/tests/AsyncCrawlerRunner/simple_default_reactor.py @@ -0,0 +1,24 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import AsyncCrawlerRunner +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@deferred_f_from_coro_f +async def main(reactor): + configure_logging() + runner = AsyncCrawlerRunner() + await runner.crawl(NoRequestsSpider) + + +react(main) diff --git a/tests/CrawlerRunner/explicit_default_reactor.py b/tests/CrawlerRunner/explicit_default_reactor.py new file mode 100644 index 000000000..9eb8a39bb --- /dev/null +++ b/tests/CrawlerRunner/explicit_default_reactor.py @@ -0,0 +1,28 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + + +class NoRequestsSpider(Spider): + name = "no_request" + + custom_settings = { + "TWISTED_REACTOR": None, + } + + async def start(self): + return + yield + + +def main(reactor): + configure_logging( + {"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"} + ) + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +react(main) diff --git a/tests/CrawlerRunner/multi_parallel.py b/tests/CrawlerRunner/multi_parallel.py new file mode 100644 index 000000000..51feccd0a --- /dev/null +++ b/tests/CrawlerRunner/multi_parallel.py @@ -0,0 +1,26 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + runner.crawl(NoRequestsSpider) + runner.crawl(NoRequestsSpider) + return runner.join() + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/CrawlerRunner/multi_seq.py b/tests/CrawlerRunner/multi_seq.py new file mode 100644 index 000000000..f6549be9b --- /dev/null +++ b/tests/CrawlerRunner/multi_seq.py @@ -0,0 +1,27 @@ +from twisted.internet.defer import inlineCallbacks +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +@inlineCallbacks +def main(reactor): + configure_logging() + runner = CrawlerRunner() + yield runner.crawl(NoRequestsSpider) + yield runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/CrawlerRunner/simple.py b/tests/CrawlerRunner/simple.py new file mode 100644 index 000000000..d154dcde4 --- /dev/null +++ b/tests/CrawlerRunner/simple.py @@ -0,0 +1,24 @@ +from twisted.internet.task import react + +from scrapy import Spider +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging +from scrapy.utils.reactor import install_reactor + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield + + +def main(reactor): + configure_logging() + runner = CrawlerRunner() + return runner.crawl(NoRequestsSpider) + + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +react(main) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 7a3d562e5..a1d3c02fb 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -18,11 +18,12 @@ from zope.interface.exceptions import MultipleInvalid import scrapy from scrapy import Spider -from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner +from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerProcess, CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader +from scrapy.utils.defer import deferred_from_coro from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings @@ -558,6 +559,26 @@ class TestCrawlerRunner(TestBaseCrawler): self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") +class TestAsyncCrawlerRunner(TestBaseCrawler): + def test_spider_manager_verify_interface(self): + settings = Settings( + { + "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, + } + ) + with pytest.raises(MultipleInvalid): + AsyncCrawlerRunner(settings) + + def test_crawler_runner_accepts_dict(self): + runner = AsyncCrawlerRunner({"foo": "bar"}) + assert runner.settings["foo"] == "bar" + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + def test_crawler_runner_accepts_None(self): + runner = AsyncCrawlerRunner() + self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") + + class TestCrawlerProcess(TestBaseCrawler): def test_crawler_process_accepts_dict(self): runner = CrawlerProcess({"foo": "bar"}) @@ -587,20 +608,25 @@ class NoRequestsSpider(scrapy.Spider): @pytest.mark.usefixtures("reactor_pytest") class TestCrawlerRunnerHasSpider(unittest.TestCase): - def _runner(self): + @staticmethod + def _runner(): return CrawlerRunner(get_reactor_settings()) + @staticmethod + def _crawl(runner, spider): + return runner.crawl(spider) + @inlineCallbacks def test_crawler_runner_bootstrap_successful(self): runner = self._runner() - yield runner.crawl(NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed @inlineCallbacks def test_crawler_runner_bootstrap_successful_for_several(self): runner = self._runner() - yield runner.crawl(NoRequestsSpider) - yield runner.crawl(NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed @inlineCallbacks @@ -608,7 +634,7 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): runner = self._runner() try: - yield runner.crawl(ExceptionSpider) + yield self._crawl(runner, ExceptionSpider) except ValueError: pass else: @@ -621,13 +647,13 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): runner = self._runner() try: - yield runner.crawl(ExceptionSpider) + yield self._crawl(runner, ExceptionSpider) except ValueError: pass else: pytest.fail("Exception should be raised from spider") - yield runner.crawl(NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) assert runner.bootstrap_failed @@ -643,7 +669,7 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): Exception, match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", ): - yield runner.crawl(NoRequestsSpider) + yield self._crawl(runner, NoRequestsSpider) else: CrawlerRunner( settings={ @@ -652,6 +678,20 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): ) +@pytest.mark.only_asyncio +class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): + @staticmethod + def _runner(): + return AsyncCrawlerRunner(get_reactor_settings()) + + @staticmethod + def _crawl(runner, spider): + return deferred_from_coro(runner.crawl(spider)) + + def test_crawler_runner_asyncio_enabled_true(self): + pytest.skip("This test is only for CrawlerRunner") + + class ScriptRunnerMixin: script_dir: Path @@ -923,6 +963,48 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): script_dir = Path(__file__).parent.resolve() / "CrawlerRunner" + def test_simple(self): + log = self.run_script("simple.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_explicit_default_reactor(self): + log = self.run_script("explicit_default_reactor.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + not in log + ) + + def test_multi_parallel(self): + log = self.run_script("multi_parallel.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Spider opened.+Closing spider.+Closing spider", + log, + re.DOTALL, + ) + + def test_multi_seq(self): + log = self.run_script("multi_seq.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Closing spider.+Spider opened.+Closing spider", + log, + re.DOTALL, + ) + def test_response_ip_address(self): log = self.run_script("ip_address.py") assert "INFO: Spider closed (finished)" in log @@ -939,6 +1021,49 @@ class TestCrawlerRunnerSubprocess(ScriptRunnerMixin): assert "DEBUG: Using asyncio event loop" in log +class TestAsyncCrawlerRunnerSubprocess(ScriptRunnerMixin): + script_dir = Path(__file__).parent.resolve() / "AsyncCrawlerRunner" + + def test_simple(self): + log = self.run_script("simple.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + + def test_simple_default_reactor(self): + log = self.run_script("simple_default_reactor.py") + assert "Spider closed (finished)" not in log + assert "RuntimeError: AsyncCrawlerRunner requires AsyncioSelectorReactor" in log + + def test_multi_parallel(self): + log = self.run_script("multi_parallel.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Spider opened.+Closing spider.+Closing spider", + log, + re.DOTALL, + ) + + def test_multi_seq(self): + log = self.run_script("multi_seq.py") + assert "Spider closed (finished)" in log + assert ( + "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" + in log + ) + assert re.search( + r"Spider opened.+Closing spider.+Spider opened.+Closing spider", + log, + re.DOTALL, + ) + + @pytest.mark.parametrize( ("settings", "items"), [