diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f74575ee1..444aa3557 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -34,25 +34,25 @@ jobs: TOXENV: py - python-version: "3.13" env: - TOXENV: asyncio + TOXENV: default-reactor - python-version: pypy3.10 env: TOXENV: pypy3 # pinned deps - - python-version: 3.9.19 + - python-version: "3.9.21" env: TOXENV: pinned - - python-version: 3.9.19 + - python-version: "3.9.21" env: - TOXENV: asyncio-pinned + TOXENV: default-reactor-pinned - python-version: pypy3.10 env: TOXENV: pypy3-pinned - - python-version: 3.9.19 + - python-version: "3.9.21" env: TOXENV: extra-deps-pinned - - python-version: 3.9.19 + - python-version: "3.9.21" env: TOXENV: botocore-pinned @@ -78,7 +78,7 @@ jobs: if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') run: | sudo apt-get update - sudo apt-get install libxml2-dev libxslt-dev libjpeg-dev + sudo apt-get install libxml2-dev libxslt-dev - name: Run tests env: ${{ matrix.env }} diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 21d621240..537a01e29 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -19,7 +19,7 @@ jobs: include: - python-version: "3.9" env: - TOXENV: windows-pinned + TOXENV: py - python-version: "3.10" env: TOXENV: py @@ -34,7 +34,19 @@ jobs: TOXENV: py - python-version: "3.13" env: - TOXENV: asyncio + TOXENV: default-reactor + + # pinned deps + - python-version: "3.9.13" + env: + TOXENV: pinned + - python-version: "3.9.13" + env: + TOXENV: extra-deps-pinned + + - python-version: "3.13" + env: + TOXENV: extra-deps steps: - uses: actions/checkout@v4 diff --git a/conftest.py b/conftest.py index f33ffb1a4..9999e41d2 100644 --- a/conftest.py +++ b/conftest.py @@ -51,7 +51,7 @@ def chdir(tmpdir): def pytest_addoption(parser): parser.addoption( "--reactor", - default="default", + default="asyncio", choices=["default", "asyncio"], ) @@ -67,17 +67,17 @@ def reactor_pytest(request): @pytest.fixture(autouse=True) def only_asyncio(request, reactor_pytest): - if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio": - pytest.skip("This test is only run with --reactor=asyncio") + if request.node.get_closest_marker("only_asyncio") and reactor_pytest == "default": + pytest.skip("This test is only run without --reactor=default") @pytest.fixture(autouse=True) def only_not_asyncio(request, reactor_pytest): if ( request.node.get_closest_marker("only_not_asyncio") - and reactor_pytest == "asyncio" + and reactor_pytest != "default" ): - pytest.skip("This test is only run without --reactor=asyncio") + pytest.skip("This test is only run with --reactor=default") @pytest.fixture(autouse=True) @@ -117,7 +117,7 @@ def requires_boto3(request): def pytest_configure(config): - if config.getoption("--reactor") == "asyncio": + if config.getoption("--reactor") != "default": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 07baea071..35afdc11b 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -16,15 +16,19 @@ asyncio reactor `, you may use :mod:`asyncio` and Installing the asyncio reactor ============================== -To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to -``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``. +To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs +to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, +which is the default value. If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` reactor manually. You can do that using -:func:`~scrapy.utils.reactor.install_reactor`:: +:func:`~scrapy.utils.reactor.install_reactor`: - install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') +.. skip: next +.. code-block:: python + + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") .. _asyncio-preinstalled-reactor: @@ -144,3 +148,14 @@ Using custom asyncio loops You can also use custom asyncio event loops with the asyncio reactor. Set the :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to use it instead of the default asyncio event loop. + + +.. _disable-asyncio: + +Switching to a non-asyncio reactor +================================== + +If for some reason your code doesn't work with the asyncio reactor, you can use +a different reactor by setting the :setting:`TWISTED_REACTOR` setting to its +import path (e.g. ``'twisted.internet.epollreactor.EPollReactor'``) or to +``None``, which will use the default reactor for your platform. diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index cc1fe8703..01da53342 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you can configure some extra functions like generating thumbnails and filtering the images based on their size. -The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for +The Images Pipeline requires Pillow_ 8.0.0 or greater. It is used for thumbnailing and normalizing images to JPEG/RGB format. .. _Pillow: https://github.com/python-pillow/Pillow diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index b2bb7148f..ca0af569f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1911,7 +1911,7 @@ TWISTED_REACTOR .. versionadded:: 2.0 -Default: ``None`` +Default: ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"`` Import path of a given :mod:`~twisted.internet.reactor`. @@ -1996,17 +1996,19 @@ which raises :exc:`Exception`, becomes: self.crawler.engine.close_spider(self, "timeout") -The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which -means that Scrapy will use the existing reactor if one is already installed, or -install the default reactor defined by Twisted for the current platform. This -is to maintain backward compatibility and avoid possible problems caused by -using a non-default reactor. +If this setting is set ``None``, Scrapy will use the existing reactor if one is +already installed, or install the default reactor defined by Twisted for the +current platform. .. versionchanged:: 2.7 The :command:`startproject` command now sets this setting to ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated ``settings.py`` file. +.. versionchanged:: VERSION + The default value was changed from ``None`` to + ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. + For additional information, see :doc:`core/howto/choosing-reactor`. diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 29dc13f0a..63c6908dc 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -68,7 +68,7 @@ class ImagesPipeline(FilesPipeline): self._Image = Image except ImportError: raise NotConfigured( - "ImagesPipeline requires installing Pillow 4.0.0 or later" + "ImagesPipeline requires installing Pillow 8.0.0 or later" ) super().__init__( diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index c473b369c..645e50301 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -341,7 +341,7 @@ TELNETCONSOLE_HOST = "127.0.0.1" TELNETCONSOLE_USERNAME = "scrapy" TELNETCONSOLE_PASSWORD = None -TWISTED_REACTOR = None +TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" SPIDER_CONTRACTS = {} SPIDER_CONTRACTS_BASE = { diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 0bb31ffaa..db7400af8 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -90,5 +90,4 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" # Set settings whose default value is deprecated to a future-proof value -TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" FEED_EXPORT_ENCODING = "utf-8" diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index b865cf48d..24e17ecb6 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -182,11 +182,9 @@ def log_scrapy_info(settings: Settings) -> None: def log_reactor_info() -> None: - from twisted.internet import reactor + from twisted.internet import asyncioreactor, reactor logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__) - from twisted.internet import asyncioreactor - if isinstance(reactor, asyncioreactor.AsyncioSelectorReactor): logger.debug( "Using asyncio event loop: %s.%s", diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index b69f43438..2da526cd8 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -18,6 +18,7 @@ from twisted.trial.unittest import SkipTest from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available from scrapy.utils.deprecate import create_deprecated_class +from scrapy.utils.reactor import is_asyncio_reactor_installed from scrapy.utils.spider import DefaultSpider if TYPE_CHECKING: @@ -109,6 +110,19 @@ def get_ftp_content_and_delete( TestSpider = create_deprecated_class("TestSpider", DefaultSpider) +def get_reactor_settings() -> dict[str, Any]: + """Return a settings dict that works with the installed reactor. + + ``Crawler._apply_settings()`` checks that the installed reactor matches the + settings, so tests that run the crawler in the current process may need to + pass a correct ``"TWISTED_REACTOR"`` setting value when creating it. + """ + settings: dict[str, Any] = {} + if not is_asyncio_reactor_installed(): + settings["TWISTED_REACTOR"] = None + return settings + + def get_crawler( spidercls: type[Spider] | None = None, settings_dict: dict[str, Any] | None = None, @@ -120,9 +134,12 @@ def get_crawler( """ from scrapy.crawler import CrawlerRunner - # Set by default settings that prevent deprecation warnings. - settings: dict[str, Any] = {} - settings.update(settings_dict or {}) + # When needed, useful settings can be added here, e.g. ones that prevent + # deprecation warnings. + settings: dict[str, Any] = { + **get_reactor_settings(), + **(settings_dict or {}), + } runner = CrawlerRunner(settings) crawler = runner.create_crawler(spidercls or DefaultSpider) crawler._apply_settings() diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index f013eed27..0c380610d 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,14 +1,8 @@ -import asyncio -import sys +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.utils.reactor import install_reactor -from twisted.internet import asyncioreactor - -if sys.platform == "win32": - asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncioreactor.install(asyncio.get_event_loop()) - -import scrapy # noqa: E402 -from scrapy.crawler import CrawlerProcess # noqa: E402 +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") class NoRequestsSpider(scrapy.Spider): diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py index 078cb72cb..e2933338b 100644 --- a/tests/CrawlerProcess/reactor_default.py +++ b/tests/CrawlerProcess/reactor_default.py @@ -1,4 +1,5 @@ from twisted.internet import reactor # noqa: F401 +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -13,5 +14,6 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess(settings={}) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py index 814a2a46d..b61e52625 100644 --- a/tests/CrawlerProcess/reactor_select.py +++ b/tests/CrawlerProcess/reactor_select.py @@ -1,4 +1,5 @@ from twisted.internet import selectreactor +from twisted.python import log import scrapy from scrapy.crawler import CrawlerProcess @@ -15,5 +16,6 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess(settings={}) -process.crawl(NoRequestsSpider) +d = process.crawl(NoRequestsSpider) +d.addErrback(log.err) process.start() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 2f1bb7713..b1b297777 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -1,3 +1,9 @@ +# ruff: noqa: E402 + +from scrapy.utils.reactor import install_reactor + +install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + from urllib.parse import urlparse from twisted.internet import reactor diff --git a/tests/test_addons.py b/tests/test_addons.py index 686bf9952..b4294c815 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -9,7 +9,7 @@ from scrapy import Spider from scrapy.crawler import Crawler, CrawlerRunner from scrapy.exceptions import NotConfigured from scrapy.settings import BaseSettings, Settings -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings class SimpleAddon: @@ -105,6 +105,7 @@ class TestAddonManager(unittest.TestCase): } settings_dict = { "ADDONS": {get_addon_cls(config): 1}, + **get_reactor_settings(), } crawler = get_crawler(settings_dict=settings_dict) assert crawler.settings.getint("KEY") == 15 @@ -119,6 +120,7 @@ class TestAddonManager(unittest.TestCase): settings_dict = { "KEY": 20, # priority=project "ADDONS": {get_addon_cls(config): 1}, + **get_reactor_settings(), } settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") @@ -196,6 +198,7 @@ class TestAddonManager(unittest.TestCase): return spider settings = Settings() + settings.setdict(get_reactor_settings()) settings.set("KEY", "default", priority="default") runner = CrawlerRunner(settings) crawler = runner.create_crawler(MySpider) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 6f4045fc8..f49deac1f 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -18,7 +18,7 @@ from scrapy.exceptions import StopDownload from scrapy.http import Request from scrapy.http.response import Response from scrapy.utils.python import to_unicode -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings from tests import NON_EXISTING_RESOLVABLE from tests.mockserver import MockServer from tests.spiders import ( @@ -412,7 +412,7 @@ with multiples lines @defer.inlineCallbacks def test_crawl_multiple(self): - runner = CrawlerRunner() + runner = CrawlerRunner(get_reactor_settings()) runner.crawl( SimpleSpider, self.mockserver.url("/status?n=200"), diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0bbcc0843..98352b66e 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -25,7 +25,7 @@ from scrapy.settings import Settings, default_settings from scrapy.spiderloader import SpiderLoader from scrapy.utils.log import configure_logging, get_scrapy_root_handler from scrapy.utils.spider import DefaultSpider -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings from tests.mockserver import MockServer, get_mockserver_env BASE_SETTINGS: dict[str, Any] = {} @@ -35,6 +35,7 @@ def get_raw_crawler(spidercls=None, settings_dict=None): """get_crawler alternative that only calls the __init__ method of the crawler.""" settings = Settings() + settings.setdict(get_reactor_settings()) settings.setdict(settings_dict or {}) return Crawler(spidercls or DefaultSpider, settings) @@ -48,7 +49,12 @@ class TestBaseCrawler(unittest.TestCase): class TestCrawler(TestBaseCrawler): def test_populate_spidercls_settings(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {**BASE_SETTINGS, "TEST1": "project", "TEST3": "project"} + project_settings = { + **BASE_SETTINGS, + "TEST1": "project", + "TEST3": "project", + **get_reactor_settings(), + } class CustomSettingsSpider(DefaultSpider): custom_settings = spider_settings @@ -581,7 +587,7 @@ class NoRequestsSpider(scrapy.Spider): @pytest.mark.usefixtures("reactor_pytest") class TestCrawlerRunnerHasSpider(unittest.TestCase): def _runner(self): - return CrawlerRunner() + return CrawlerRunner(get_reactor_settings()) @inlineCallbacks def test_crawler_runner_bootstrap_successful(self): @@ -626,13 +632,7 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): @inlineCallbacks def test_crawler_runner_asyncio_enabled_true(self): - if self.reactor_pytest == "asyncio": - CrawlerRunner( - settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - } - ) - else: + if self.reactor_pytest == "default": runner = CrawlerRunner( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", @@ -643,6 +643,12 @@ class TestCrawlerRunnerHasSpider(unittest.TestCase): match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", ): yield runner.crawl(NoRequestsSpider) + else: + CrawlerRunner( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + ) class ScriptRunnerMixin: @@ -672,7 +678,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): assert "Spider closed (finished)" in log assert ( "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log + in log ) def test_multi(self): @@ -680,18 +686,17 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): assert "Spider closed (finished)" in log assert ( "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log + in log ) assert "ReactorAlreadyInstalledError" not in log def test_reactor_default(self): log = self.run_script("reactor_default.py") - assert "Spider closed (finished)" in log + assert "Spider closed (finished)" not in log assert ( - "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" - not in log - ) - assert "ReactorAlreadyInstalledError" not in log + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log def test_reactor_default_twisted_reactor_select(self): log = self.run_script("reactor_default_twisted_reactor_select.py") @@ -716,8 +721,11 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): def test_reactor_select(self): log = self.run_script("reactor_select.py") - assert "Spider closed (finished)" in log - assert "ReactorAlreadyInstalledError" not in log + assert "Spider closed (finished)" not in log + assert ( + "does not match the requested one " + "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" + ) in log def test_reactor_select_twisted_reactor_select(self): log = self.run_script("reactor_select_twisted_reactor_select.py") diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 162747581..c2df67c66 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -33,7 +33,7 @@ class TestScrapyUtils: tox_config_file_path = Path(__file__).parent / ".." / "tox.ini" config_parser = ConfigParser() config_parser.read(tox_config_file_path) - pattern = r"Twisted\[http2\]==([\d.]+)" + pattern = r"Twisted==([\d.]+)" match = re.search(pattern, config_parser["pinned"]["deps"]) pinned_twisted_version_string = match[1] diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 19bd02498..bc18e76e1 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -307,7 +307,7 @@ class TestHttp(unittest.TestCase, ABC): @defer.inlineCallbacks def test_timeout_download_from_spider_nodata_rcvd(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": + if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( "This test produces DirtyReactorAggregateError on Windows with asyncio" @@ -322,7 +322,7 @@ class TestHttp(unittest.TestCase, ABC): @defer.inlineCallbacks def test_timeout_download_from_spider_server_hangs(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": + if self.reactor_pytest != "default" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 raise unittest.SkipTest( "This test produces DirtyReactorAggregateError on Windows with asyncio" @@ -1136,7 +1136,7 @@ class TestFTPBase(unittest.TestCase): class TestFTP(TestFTPBase): def test_invalid_credentials(self): - if self.reactor_pytest == "asyncio" and sys.platform == "win32": + if self.reactor_pytest != "default" and sys.platform == "win32": raise unittest.SkipTest( "This test produces DirtyReactorAggregateError on Windows with asyncio" ) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 15b3ad5af..4fca9eefb 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -64,7 +64,7 @@ class CrawlTestCase(TestCase): @defer.inlineCallbacks def test_delay(self): - crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider) + crawler = get_crawler(DownloaderSlotsSettingsTestSpider) yield crawler.crawl(mockserver=self.mockserver) slots = crawler.engine.downloader.slots times = crawler.spider.times diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index ca5ffdc26..85bd42857 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -1,9 +1,11 @@ -import datetime -import typing -import unittest +from __future__ import annotations + +import datetime +import unittest +from typing import Any, Callable -from scrapy.crawler import Crawler from scrapy.extensions.periodic_log import PeriodicLog +from scrapy.utils.test import get_crawler from .spiders import MetaSpider @@ -59,9 +61,8 @@ class CustomPeriodicLog(PeriodicLog): self.stats._stats = stats_dump_2 -def extension(settings=None): - crawler = Crawler(MetaSpider, settings=settings) - crawler._apply_settings() +def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog: + crawler = get_crawler(MetaSpider, settings) return CustomPeriodicLog.from_crawler(crawler) @@ -94,7 +95,7 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict, condition: typing.Callable): + def check(settings: dict[str, Any], condition: Callable) -> None: ext, a, b = emulate(settings) assert list(a["delta"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) @@ -151,7 +152,7 @@ class TestPeriodicLog(unittest.TestCase): ext.spider_closed(spider, reason="finished") return ext, a, b - def check(settings: dict, condition: typing.Callable): + def check(settings: dict[str, Any], condition: Callable) -> None: ext, a, b = emulate(settings) assert list(a["stats"].keys()) == [ k for k, v in ext.stats._stats.items() if condition(k, v) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 162dfdaf4..c5f1b6321 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -3,18 +3,22 @@ from __future__ import annotations import shutil from pathlib import Path from tempfile import mkdtemp +from typing import TYPE_CHECKING, Any from testfixtures import LogCapture from twisted.internet import defer from twisted.trial.unittest import TestCase from w3lib.url import add_or_replace_parameter -from scrapy import signals -from scrapy.crawler import CrawlerRunner +from scrapy import Spider, signals from scrapy.utils.misc import load_object +from scrapy.utils.test import get_crawler from tests.mockserver import MockServer from tests.spiders import SimpleSpider +if TYPE_CHECKING: + from scrapy.crawler import Crawler + class MediaDownloadSpider(SimpleSpider): name = "mediadownload" @@ -80,7 +84,6 @@ class TestFileDownloadCrawl(TestCase): "ITEM_PIPELINES": {self.pipeline_class: 1}, self.store_setting_key: str(self.tmpmediastore), } - self.runner = CrawlerRunner(self.settings) self.items = [] def tearDown(self): @@ -90,10 +93,12 @@ class TestFileDownloadCrawl(TestCase): def _on_item_scraped(self, item): self.items.append(item) - def _create_crawler(self, spider_class, runner=None, **kwargs): - if runner is None: - runner = self.runner - crawler = runner.create_crawler(spider_class, **kwargs) + def _create_crawler( + self, spider_class: type[Spider], settings: dict[str, Any] | None = None + ) -> Crawler: + if settings is None: + settings = self.settings + crawler = get_crawler(spider_class, settings) crawler.signals.connect(self._on_item_scraped, signals.item_scraped) return crawler @@ -175,10 +180,11 @@ class TestFileDownloadCrawl(TestCase): @defer.inlineCallbacks def test_download_media_redirected_allowed(self): - settings = dict(self.settings) - settings.update({"MEDIA_ALLOW_REDIRECTS": True}) - runner = CrawlerRunner(settings) - crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner) + settings = { + **self.settings, + "MEDIA_ALLOW_REDIRECTS": True, + } + crawler = self._create_crawler(RedirectedMediaDownloadSpider, settings) with LogCapture() as log: yield crawler.crawl( self.mockserver.url("/files/images/"), @@ -201,8 +207,7 @@ class TestFileDownloadCrawl(TestCase): **self.settings, "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1}, } - runner = CrawlerRunner(settings) - crawler = self._create_crawler(MediaDownloadSpider, runner=runner) + crawler = self._create_crawler(MediaDownloadSpider, settings) with LogCapture() as log: yield crawler.crawl( self.mockserver.url("/files/images/"), diff --git a/tests/test_spider.py b/tests/test_spider.py index 4e8330c06..aaf72390d 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -27,7 +27,7 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.spiders.init import InitSpider -from scrapy.utils.test import get_crawler +from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata, tests_datadir @@ -108,7 +108,11 @@ class TestSpider(unittest.TestCase): @inlineCallbacks def test_settings_in_from_crawler(self): spider_settings = {"TEST1": "spider", "TEST2": "spider"} - project_settings = {"TEST1": "project", "TEST3": "project"} + project_settings = { + "TEST1": "project", + "TEST3": "project", + **get_reactor_settings(), + } class TestSpider(self.spider_class): name = "test" diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index a65a36219..901e03d59 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -2,6 +2,7 @@ import asyncio import warnings import pytest +from twisted.trial.unittest import TestCase from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( @@ -12,10 +13,10 @@ from scrapy.utils.reactor import ( @pytest.mark.usefixtures("reactor_pytest") -class TestAsyncio: +class TestAsyncio(TestCase): def test_is_asyncio_reactor_installed(self): # the result should depend only on the pytest --reactor argument - assert is_asyncio_reactor_installed() == (self.reactor_pytest == "asyncio") + assert is_asyncio_reactor_installed() == (self.reactor_pytest != "default") def test_install_asyncio_reactor(self): from twisted.internet import reactor as original_reactor diff --git a/tox.ini b/tox.ini index 70c841603..eb084f0f5 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ deps = {[test-requirements]deps} # mitmproxy does not support PyPy - mitmproxy; implementation_name != 'pypy' + mitmproxy; implementation_name != "pypy" setenv = COVERAGE_CORE=sysmon passenv = @@ -96,19 +96,18 @@ commands = [pinned] basepython = python3.9 deps = + Protego==0.1.15 + Twisted==21.7.0 cryptography==37.0.0 cssselect==0.9.1 - h2==3.0 itemadapter==0.1.0 + lxml==4.6.0 parsel==1.5.0 - Protego==0.1.15 pyOpenSSL==22.0.0 queuelib==1.4.2 service_identity==18.1.0 - Twisted[http2]==21.7.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.6.0 {[test-requirements]deps} # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies @@ -131,60 +130,50 @@ setenv = {[pinned]setenv} commands = {[pinned]commands} -[testenv:windows-pinned] -basepython = {[pinned]basepython} -deps = - {[pinned]deps} - PyDispatcher==2.0.5 -install_command = {[pinned]install_command} -setenv = - {[pinned]setenv} -commands = {[pinned]commands} - [testenv:extra-deps] basepython = python3 deps = {[testenv]deps} - boto3 - google-cloud-storage - robotexclusionrulesparser Pillow Twisted[http2] - uvloop; platform_system != "Windows" + boto3 bpython # optional for shell wrapper tests - brotli; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests - brotlicffi; implementation_name == 'pypy' # optional for HTTP compress downloader middleware tests - zstandard; implementation_name != 'pypy' # optional for HTTP compress downloader middleware tests + brotli; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests + brotlicffi; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests + google-cloud-storage ipython + robotexclusionrulesparser + uvloop; platform_system != "Windows" + zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests [testenv:extra-deps-pinned] basepython = {[pinned]basepython} deps = {[pinned]deps} + Pillow==8.0.0 boto3==1.20.0 - google-cloud-storage==1.29.0 - Pillow==7.1.0 - robotexclusionrulesparser==1.6.2 - brotlipy - uvloop==0.14.0; platform_system != "Windows" bpython==0.7.1 - zstandard==0.1; implementation_name != 'pypy' + brotli==0.5.2; implementation_name != "pypy" + brotlicffi==0.8.0; implementation_name == "pypy" + brotlipy + google-cloud-storage==1.29.0 ipython==2.0.0 - brotli==0.5.2; implementation_name != 'pypy' - brotlicffi==0.8.0; implementation_name == 'pypy' + robotexclusionrulesparser==1.6.2 + uvloop==0.14.0; platform_system != "Windows" + zstandard==0.1; implementation_name != "pypy" install_command = {[pinned]install_command} setenv = {[pinned]setenv} commands = {[pinned]commands} -[testenv:asyncio] +[testenv:default-reactor] commands = - {[testenv]commands} --reactor=asyncio + {[testenv]commands} --reactor=default -[testenv:asyncio-pinned] +[testenv:default-reactor-pinned] basepython = {[pinned]basepython} deps = {[testenv:pinned]deps} -commands = {[pinned]commands} --reactor=asyncio +commands = {[pinned]commands} --reactor=default install_command = {[pinned]install_command} setenv = {[pinned]setenv} @@ -204,21 +193,20 @@ commands = {[testenv:pypy3]commands} [testenv:pypy3-pinned] basepython = pypy3.10 deps = + PyPyDispatcher==2.1.0 + {[test-requirements]deps} + Protego==0.1.15 + Twisted==21.7.0 cryptography==41.0.5 cssselect==0.9.1 - h2==3.1 itemadapter==0.1.0 + lxml==4.6.0 parsel==1.5.0 - Protego==0.1.15 pyOpenSSL==23.3.0 queuelib==1.4.2 service_identity==18.1.0 - Twisted[http2]==21.7.0 w3lib==1.17.0 zope.interface==5.1.0 - lxml==4.6.0 - {[test-requirements]deps} - PyPyDispatcher==2.1.0 commands = ; disabling both coverage and docs tests pytest {posargs:--durations=10 scrapy tests}