diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 9c324a031..c82e6c2f3 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -81,8 +81,8 @@ These APIs have a coroutine-based implementation and a Deferred-based one: - :meth:`~scrapy.crawler.Crawler.crawl_async` (coroutine-based) and :meth:`~scrapy.crawler.Crawler.crawl` (Deferred-based): the former - doesn't support non-default reactors and so the latter should be used - with those. + may be inconvenient to use in Deferred-based code so both are available, + this may change in a future Scrapy version. - :class:`scrapy.crawler.AsyncCrawlerRunner` and its subclass :class:`scrapy.crawler.AsyncCrawlerProcess` (coroutine-based) and diff --git a/scrapy/crawler.py b/scrapy/crawler.py index ffbebe152..e21207222 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -19,7 +19,6 @@ from scrapy.extension import ExtensionManager from scrapy.settings import Settings, overridden_settings from scrapy.signalmanager import SignalManager from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader -from scrapy.utils.asyncio import is_asyncio_available from scrapy.utils.defer import deferred_from_coro from scrapy.utils.log import ( configure_logging, @@ -169,13 +168,7 @@ class Crawler: .. versionadded:: VERSION Complete when the crawl is finished. - - This function requires - :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be - installed. """ - if not is_asyncio_available(): - raise RuntimeError("Crawler.crawl_async() requires AsyncioSelectorReactor.") if self.crawling: raise RuntimeError("Crawling already taking place") if self._started: diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index a84bb9d8d..a35bfd590 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -187,4 +187,4 @@ async def test_fetch_deprecated_spider_arg(): ScrapyDeprecationWarning, match=r"The fetch\(\) method of .+\.CustomDownloader requires a spider argument", ): - await maybe_deferred_to_future(crawler.crawl()) + await crawler.crawl_async() diff --git a/tests/test_core_scraper.py b/tests/test_core_scraper.py index c819e246e..47dee1271 100644 --- a/tests/test_core_scraper.py +++ b/tests/test_core_scraper.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider @@ -23,5 +23,5 @@ async def test_scraper_exception( "scrapy.core.engine.Scraper.handle_spider_output_async", lambda *args, **kwargs: 1 / 0, ) - await maybe_deferred_to_future(crawler.crawl(url=mockserver.url("/"))) + await crawler.crawl_async(url=mockserver.url("/")) assert "Scraper bug processing" in caplog.text diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b0060a76a..699ac3911 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -98,7 +98,7 @@ class TestCrawl: settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize} crawler = get_crawler(FollowAllSpider, settings) - await maybe_deferred_to_future(crawler.crawl(**crawl_kwargs)) + await crawler.crawl_async(**crawl_kwargs) assert crawler.spider assert isinstance(crawler.spider, FollowAllSpider) times = crawler.spider.times @@ -112,7 +112,7 @@ class TestCrawl: # code above to have any meaning. settings["DOWNLOAD_DELAY"] = 0 crawler = get_crawler(FollowAllSpider, settings) - await maybe_deferred_to_future(crawler.crawl(**crawl_kwargs)) + await crawler.crawl_async(**crawl_kwargs) assert crawler.spider assert isinstance(crawler.spider, FollowAllSpider) times = crawler.spider.times diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 4f1045786..5204c9a73 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -111,7 +111,6 @@ class TestCrawler(TestBaseCrawler): with pytest.raises(RuntimeError, match="more than once on the same instance"): yield crawler.crawl() - @pytest.mark.only_asyncio @deferred_f_from_coro_f async def test_crawler_crawl_async_twice_seq_unsupported(self): crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) @@ -552,7 +551,7 @@ class TestCrawlerLogging: assert get_scrapy_root_handler().level == logging.DEBUG crawler = get_crawler(MySpider) assert get_scrapy_root_handler().level == logging.INFO - await maybe_deferred_to_future(crawler.crawl()) + await crawler.crawl_async() finally: _uninstall_scrapy_root_handler() diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 5d88b59c1..22127a4da 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -8,7 +8,7 @@ from scrapy import Request from scrapy.core.downloader import Downloader, Slot from scrapy.crawler import CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer @@ -143,7 +143,7 @@ async def test_none_slot_with_priority_queue( DownloaderSlotsSettingsTestSpider, settings_dict={"SCHEDULER_PRIORITY_QUEUE": priority_queue_class}, ) - await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver)) + await crawler.crawl_async(mockserver=mockserver) assert isinstance(crawler.spider, DownloaderSlotsSettingsTestSpider) assert hasattr(crawler.spider, "times") diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index ddf1f1fe0..c219a0f6f 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -81,7 +81,7 @@ class TestMain: settings = {"SCHEDULER": MemoryScheduler} crawler = get_crawler(TestSpider, settings_dict=settings) crawler.signals.connect(track_url, signals.request_reached_downloader) - await maybe_deferred_to_future(crawler.crawl()) + await crawler.crawl_async() assert crawler.stats.get_value("finish_reason") == "finished" expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"] assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" @@ -112,7 +112,7 @@ class TestMain: caplog.clear() with caplog.at_level(ERROR): - await maybe_deferred_to_future(crawler.crawl()) + await crawler.crawl_async() assert not caplog.records assert crawler.stats @@ -182,7 +182,7 @@ class TestRequestSendOrder: crawler = get_crawler(TestSpider, settings_dict=settings) crawler.signals.connect(track_num, signals.request_reached_downloader) - await maybe_deferred_to_future(crawler.crawl()) + await crawler.crawl_async() assert crawler.stats.get_value("finish_reason") == "finished" expected_nums = sorted(start_nums + cb_nums) assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}" diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index a423f5328..a9d608e52 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -862,7 +862,7 @@ class TestFeedExport(TestFeedExportBase): try: spider_cls.start_urls = [self.mockserver.url("/")] crawler = get_crawler(spider_cls, settings) - await maybe_deferred_to_future(crawler.crawl()) + await crawler.crawl_async() for file_path, feed_options in FEEDS.items(): content[feed_options["format"]] = ( @@ -1866,7 +1866,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): try: spider_cls.start_urls = [self.mockserver.url("/")] crawler = get_crawler(spider_cls, settings) - await maybe_deferred_to_future(crawler.crawl()) + await crawler.crawl_async() for file_path in FEEDS: content[str(file_path)] = ( @@ -2363,7 +2363,7 @@ class TestBatchDeliveries(TestFeedExportBase): content: defaultdict[str, list[bytes]] = defaultdict(list) spider_cls.start_urls = [self.mockserver.url("/")] crawler = get_crawler(spider_cls, settings) - await maybe_deferred_to_future(crawler.crawl()) + await crawler.crawl_async() for path, feed in FEEDS.items(): dir_name = Path(path).parent diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index fc61d61ec..f315388cd 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -159,7 +159,7 @@ class TestPipeline: @deferred_f_from_coro_f async def test_pipeline(self, mockserver: MockServer, pipeline_class: type) -> None: crawler = self._create_crawler(pipeline_class) - await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver)) + await crawler.crawl_async(mockserver=mockserver) assert len(self.items) == 1 @deferred_f_from_coro_f @@ -179,7 +179,7 @@ class TestPipeline: match="DeferredPipeline.process_item returned a Deferred", ), ): - await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver)) + await crawler.crawl_async(mockserver=mockserver) assert len(self.items) == 1 @deferred_f_from_coro_f @@ -199,7 +199,7 @@ class TestPipeline: match=r"DeprecatedSpiderArgPipeline.process_item\(\) requires a spider argument", ), ): - await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver)) + await crawler.crawl_async(mockserver=mockserver) assert len(self.items) == 1 @@ -224,7 +224,7 @@ class TestPipeline: pipeline_class: type, ) -> None: crawler = self._create_crawler(pipeline_class) - await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver)) + await crawler.crawl_async(mockserver=mockserver) assert "Error processing {'field': 42}" in caplog.text assert "process_item error" in caplog.text @@ -247,7 +247,7 @@ class TestPipeline: ) -> None: crawler = self._create_crawler(pipeline_class) with pytest.raises(ValueError, match="open_spider error"): - await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver)) + await crawler.crawl_async(mockserver=mockserver) class TestCustomPipelineManager: @@ -287,7 +287,7 @@ class TestCustomPipelineManager: ) crawler.spider = crawler._create_spider() crawler.signals.connect(_on_item_scraped, signals.item_scraped) - await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver)) + await crawler.crawl_async(mockserver=mockserver) assert len(items) == 1 @@ -345,7 +345,7 @@ class TestCustomPipelineManager: match=r"CustomPipelineManager overrides process_item\(\) but doesn't override process_item_async\(\)", ), ): - await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver)) + await crawler.crawl_async(mockserver=mockserver) assert len(items) == 1 @@ -407,7 +407,7 @@ class TestCustomPipelineManager: match=r"CustomPipelineManager doesn't define a process_item_async\(\) method", ), ): - await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver)) + await crawler.crawl_async(mockserver=mockserver) assert len(items) == 1 diff --git a/tests/test_signals.py b/tests/test_signals.py index 2e4f9ffb1..89b7a74dc 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -2,7 +2,7 @@ import pytest from twisted.internet.defer import inlineCallbacks from scrapy import Request, Spider, signals -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.mockserver.http import MockServer @@ -30,7 +30,7 @@ class TestMain: calls.append(object()) crawler.signals.connect(track_call, signals.scheduler_empty) - await maybe_deferred_to_future(crawler.crawl()) + await crawler.crawl_async() assert len(calls) >= 1 diff --git a/tests/test_spider.py b/tests/test_spider.py index dcc9d1e82..a1649aa6f 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -29,7 +29,7 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.spiders.init import InitSpider -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata, tests_datadir @@ -155,7 +155,7 @@ class TestInitSpider(TestSpider): responses.append(response) crawler = get_crawler(TestSpider) - await maybe_deferred_to_future(crawler.crawl()) + await crawler.crawl_async() assert len(responses) == 1 assert responses[0].url == "data:," diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index d4eca85b8..e608d8975 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -32,7 +32,7 @@ class TestMain: crawler = get_crawler(spider) crawler.signals.connect(track_item, signals.item_scraped) - await maybe_deferred_to_future(crawler.crawl()) + await crawler.crawl_async() assert crawler.stats assert crawler.stats.get_value("finish_reason") == "finished" assert actual_items == expected_items diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 1808e087c..97d9d6470 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -1,7 +1,7 @@ from testfixtures import LogCapture from scrapy import Request, Spider -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer @@ -333,7 +333,7 @@ class TestSpiderMiddleware: async def crawl_log(self, spider: type[Spider]) -> LogCapture: crawler = get_crawler(spider) with LogCapture() as log: - await maybe_deferred_to_future(crawler.crawl(mockserver=self.mockserver)) + await crawler.crawl_async(mockserver=self.mockserver) return log @deferred_f_from_coro_f diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index a525f991d..18868bf8c 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -117,7 +117,7 @@ class TestMain: } crawler = get_crawler(spider_cls, settings_dict=settings) crawler.signals.connect(track_item, signals.item_scraped) - await maybe_deferred_to_future(crawler.crawl()) + await crawler.crawl_async() assert crawler.stats.get_value("finish_reason") == "finished" assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"