RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow Crawler.crawl_async() without asyncio. (#7202)

This commit is contained in:
Andrey Rakhmatullin 2025-12-31 22:33:59 +05:00 committed by GitHub
parent 9381ad893d
commit a2463325db
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 32 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docs/topics/coroutines.rst
View File

@ -81,8 +81,8 @@ These APIs have a coroutine-based implementation and a Deferred-based one:
- :meth:`~scrapy.crawler.Crawler.crawl_async` (coroutine-based) and
:meth:`~scrapy.crawler.Crawler.crawl` (Deferred-based): the former
doesn't support non-default reactors and so the latter should be used
with those.
may be inconvenient to use in Deferred-based code so both are available,
this may change in a future Scrapy version.
- :class:`scrapy.crawler.AsyncCrawlerRunner` and its subclass
:class:`scrapy.crawler.AsyncCrawlerProcess` (coroutine-based) and

7
scrapy/crawler.py
View File

@ -19,7 +19,6 @@ from scrapy.extension import ExtensionManager
from scrapy.settings import Settings, overridden_settings
from scrapy.signalmanager import SignalManager
from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.defer import deferred_from_coro
from scrapy.utils.log import (
configure_logging,
@ -169,13 +168,7 @@ class Crawler:
.. versionadded:: VERSION
Complete when the crawl is finished.
This function requires
:class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be
installed.
"""
if not is_asyncio_available():
raise RuntimeError("Crawler.crawl_async() requires AsyncioSelectorReactor.")
if self.crawling:
raise RuntimeError("Crawling already taking place")
if self._started:

2
tests/test_core_downloader.py
View File

@ -187,4 +187,4 @@ async def test_fetch_deprecated_spider_arg():
ScrapyDeprecationWarning,
match=r"The fetch\(\) method of .+\.CustomDownloader requires a spider argument",
):
await maybe_deferred_to_future(crawler.crawl())
await crawler.crawl_async()

4
tests/test_core_scraper.py
View File

@ -2,7 +2,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.test import get_crawler
from tests.spiders import SimpleSpider
@ -23,5 +23,5 @@ async def test_scraper_exception(
"scrapy.core.engine.Scraper.handle_spider_output_async",
lambda *args, **kwargs: 1 / 0,
)
await maybe_deferred_to_future(crawler.crawl(url=mockserver.url("/")))
await crawler.crawl_async(url=mockserver.url("/"))
assert "Scraper bug processing" in caplog.text

4
tests/test_crawl.py
View File

@ -98,7 +98,7 @@ class TestCrawl:
settings = {"DOWNLOAD_DELAY": delay, "RANDOMIZE_DOWNLOAD_DELAY": randomize}
crawler = get_crawler(FollowAllSpider, settings)
await maybe_deferred_to_future(crawler.crawl(**crawl_kwargs))
await crawler.crawl_async(**crawl_kwargs)
assert crawler.spider
assert isinstance(crawler.spider, FollowAllSpider)
times = crawler.spider.times
@ -112,7 +112,7 @@ class TestCrawl:
# code above to have any meaning.
settings["DOWNLOAD_DELAY"] = 0
crawler = get_crawler(FollowAllSpider, settings)
await maybe_deferred_to_future(crawler.crawl(**crawl_kwargs))
await crawler.crawl_async(**crawl_kwargs)
assert crawler.spider
assert isinstance(crawler.spider, FollowAllSpider)
times = crawler.spider.times

3
tests/test_crawler.py
View File

@ -111,7 +111,6 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError, match="more than once on the same instance"):
yield crawler.crawl()
@pytest.mark.only_asyncio
@deferred_f_from_coro_f
async def test_crawler_crawl_async_twice_seq_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
@ -552,7 +551,7 @@ class TestCrawlerLogging:
assert get_scrapy_root_handler().level == logging.DEBUG
crawler = get_crawler(MySpider)
assert get_scrapy_root_handler().level == logging.INFO
await maybe_deferred_to_future(crawler.crawl())
await crawler.crawl_async()
finally:
_uninstall_scrapy_root_handler()

4
tests/test_downloaderslotssettings.py
View File

@ -8,7 +8,7 @@ from scrapy import Request
from scrapy.core.downloader import Downloader, Slot
from scrapy.crawler import CrawlerRunner
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
@ -143,7 +143,7 @@ async def test_none_slot_with_priority_queue(
DownloaderSlotsSettingsTestSpider,
settings_dict={"SCHEDULER_PRIORITY_QUEUE": priority_queue_class},
)
await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
await crawler.crawl_async(mockserver=mockserver)
assert isinstance(crawler.spider, DownloaderSlotsSettingsTestSpider)
assert hasattr(crawler.spider, "times")

6
tests/test_engine_loop.py
View File

@ -81,7 +81,7 @@ class TestMain:
settings = {"SCHEDULER": MemoryScheduler}
crawler = get_crawler(TestSpider, settings_dict=settings)
crawler.signals.connect(track_url, signals.request_reached_downloader)
await maybe_deferred_to_future(crawler.crawl())
await crawler.crawl_async()
assert crawler.stats.get_value("finish_reason") == "finished"
expected_urls = ["data:,a", "data:,b", "data:,c", "data:,d"]
assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"
@ -112,7 +112,7 @@ class TestMain:
caplog.clear()
with caplog.at_level(ERROR):
await maybe_deferred_to_future(crawler.crawl())
await crawler.crawl_async()
assert not caplog.records
assert crawler.stats
@ -182,7 +182,7 @@ class TestRequestSendOrder:
crawler = get_crawler(TestSpider, settings_dict=settings)
crawler.signals.connect(track_num, signals.request_reached_downloader)
await maybe_deferred_to_future(crawler.crawl())
await crawler.crawl_async()
assert crawler.stats.get_value("finish_reason") == "finished"
expected_nums = sorted(start_nums + cb_nums)
assert actual_nums == expected_nums, f"{actual_nums=} != {expected_nums=}"

6
tests/test_feedexport.py
View File

@ -862,7 +862,7 @@ class TestFeedExport(TestFeedExportBase):
try:
spider_cls.start_urls = [self.mockserver.url("/")]
crawler = get_crawler(spider_cls, settings)
await maybe_deferred_to_future(crawler.crawl())
await crawler.crawl_async()
for file_path, feed_options in FEEDS.items():
content[feed_options["format"]] = (
@ -1866,7 +1866,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
try:
spider_cls.start_urls = [self.mockserver.url("/")]
crawler = get_crawler(spider_cls, settings)
await maybe_deferred_to_future(crawler.crawl())
await crawler.crawl_async()
for file_path in FEEDS:
content[str(file_path)] = (
@ -2363,7 +2363,7 @@ class TestBatchDeliveries(TestFeedExportBase):
content: defaultdict[str, list[bytes]] = defaultdict(list)
spider_cls.start_urls = [self.mockserver.url("/")]
crawler = get_crawler(spider_cls, settings)
await maybe_deferred_to_future(crawler.crawl())
await crawler.crawl_async()
for path, feed in FEEDS.items():
dir_name = Path(path).parent

16
tests/test_pipelines.py
View File

@ -159,7 +159,7 @@ class TestPipeline:
@deferred_f_from_coro_f
async def test_pipeline(self, mockserver: MockServer, pipeline_class: type) -> None:
crawler = self._create_crawler(pipeline_class)
await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
await crawler.crawl_async(mockserver=mockserver)
assert len(self.items) == 1
@deferred_f_from_coro_f
@ -179,7 +179,7 @@ class TestPipeline:
match="DeferredPipeline.process_item returned a Deferred",
),
):
await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
await crawler.crawl_async(mockserver=mockserver)
assert len(self.items) == 1
@deferred_f_from_coro_f
@ -199,7 +199,7 @@ class TestPipeline:
match=r"DeprecatedSpiderArgPipeline.process_item\(\) requires a spider argument",
),
):
await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
await crawler.crawl_async(mockserver=mockserver)
assert len(self.items) == 1
@ -224,7 +224,7 @@ class TestPipeline:
pipeline_class: type,
) -> None:
crawler = self._create_crawler(pipeline_class)
await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
await crawler.crawl_async(mockserver=mockserver)
assert "Error processing {'field': 42}" in caplog.text
assert "process_item error" in caplog.text
@ -247,7 +247,7 @@ class TestPipeline:
) -> None:
crawler = self._create_crawler(pipeline_class)
with pytest.raises(ValueError, match="open_spider error"):
await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
await crawler.crawl_async(mockserver=mockserver)
class TestCustomPipelineManager:
@ -287,7 +287,7 @@ class TestCustomPipelineManager:
)
crawler.spider = crawler._create_spider()
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
await crawler.crawl_async(mockserver=mockserver)
assert len(items) == 1
@ -345,7 +345,7 @@ class TestCustomPipelineManager:
match=r"CustomPipelineManager overrides process_item\(\) but doesn't override process_item_async\(\)",
),
):
await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
await crawler.crawl_async(mockserver=mockserver)
assert len(items) == 1
@ -407,7 +407,7 @@ class TestCustomPipelineManager:
match=r"CustomPipelineManager doesn't define a process_item_async\(\) method",
),
):
await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
await crawler.crawl_async(mockserver=mockserver)
assert len(items) == 1

4
tests/test_signals.py
View File

@ -2,7 +2,7 @@ import pytest
from twisted.internet.defer import inlineCallbacks
from scrapy import Request, Spider, signals
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.mockserver.http import MockServer
@ -30,7 +30,7 @@ class TestMain:
calls.append(object())
crawler.signals.connect(track_call, signals.scheduler_empty)
await maybe_deferred_to_future(crawler.crawl())
await crawler.crawl_async()
assert len(calls) >= 1

4
tests/test_spider.py
View File

@ -29,7 +29,7 @@ from scrapy.spiders import (
XMLFeedSpider,
)
from scrapy.spiders.init import InitSpider
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests import get_testdata, tests_datadir
@ -155,7 +155,7 @@ class TestInitSpider(TestSpider):
responses.append(response)
crawler = get_crawler(TestSpider)
await maybe_deferred_to_future(crawler.crawl())
await crawler.crawl_async()
assert len(responses) == 1
assert responses[0].url == "data:,"

2
tests/test_spider_start.py
View File

@ -32,7 +32,7 @@ class TestMain:
crawler = get_crawler(spider)
crawler.signals.connect(track_item, signals.item_scraped)
await maybe_deferred_to_future(crawler.crawl())
await crawler.crawl_async()
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items

4
tests/test_spidermiddleware_output_chain.py
View File

@ -1,7 +1,7 @@
from testfixtures import LogCapture
from scrapy import Request, Spider
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
@ -333,7 +333,7 @@ class TestSpiderMiddleware:
async def crawl_log(self, spider: type[Spider]) -> LogCapture:
crawler = get_crawler(spider)
with LogCapture() as log:
await maybe_deferred_to_future(crawler.crawl(mockserver=self.mockserver))
await crawler.crawl_async(mockserver=self.mockserver)
return log
@deferred_f_from_coro_f

2
tests/test_spidermiddleware_process_start.py
View File

@ -117,7 +117,7 @@ class TestMain:
}
crawler = get_crawler(spider_cls, settings_dict=settings)
crawler.signals.connect(track_item, signals.item_scraped)
await maybe_deferred_to_future(crawler.crawl())
await crawler.crawl_async()
assert crawler.stats.get_value("finish_reason") == "finished"
assert actual_items == expected_items, f"{actual_items=} != {expected_items=}"