diff --git a/tests/spiders.py b/tests/spiders.py index 612dc11c9..da14fdbe3 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -571,3 +571,19 @@ class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider): def headers_received(self, headers, body_length, request, spider): self.meta["headers_received"] = headers raise StopDownload(fail=True) + + +class ExceptionSpider(Spider): + name = "exception" + + @classmethod + def from_crawler(cls, crawler, *args, **kwargs): + raise ValueError("Exception in from_crawler method") + + +class NoRequestsSpider(Spider): + name = "no_request" + + async def start(self): + return + yield diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 815f87026..794c6d400 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -7,7 +7,7 @@ from unittest import TestCase from unittest.mock import MagicMock, Mock, PropertyMock, call, patch from scrapy.commands.check import Command, TextTestResult -from tests.test_commands import TestProjectBase +from tests.utils.base_commands import TestProjectBase from tests.utils.cmdline import proc if TYPE_CHECKING: diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py index dd69d36ba..6293e973e 100644 --- a/tests/test_command_crawl.py +++ b/tests/test_command_crawl.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import TYPE_CHECKING -from tests.test_commands import TestProjectBase +from tests.utils.base_commands import TestProjectBase from tests.utils.cmdline import proc if TYPE_CHECKING: diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py index 94db14c30..f465d0b30 100644 --- a/tests/test_command_genspider.py +++ b/tests/test_command_genspider.py @@ -6,15 +6,8 @@ from pathlib import Path import pytest -from tests.test_commands import TestProjectBase -from tests.utils.cmdline import call, proc - - -def write_recording_editor(editor: Path) -> None: - """Create an executable editor script that writes the path it is asked to - open (its last argument) into the file given as its first argument.""" - editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8") - editor.chmod(0o755) +from tests.utils.base_commands import TestProjectBase +from tests.utils.cmdline import call, proc, write_recording_editor def find_in_file(filename: Path, regex: str) -> re.Match[str] | None: diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index c210a06e8..a3581d764 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -8,7 +8,7 @@ import pytest from scrapy.commands import parse from scrapy.settings import Settings -from tests.test_commands import TestProjectBase +from tests.utils.base_commands import TestProjectBase from tests.utils.cmdline import call, proc if TYPE_CHECKING: diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py index 9dabbc942..2b410b5c6 100644 --- a/tests/test_command_runspider.py +++ b/tests/test_command_runspider.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING import pytest -from tests.test_crawler import ExceptionSpider, NoRequestsSpider +from tests.spiders import ExceptionSpider, NoRequestsSpider from tests.utils.cmdline import proc if TYPE_CHECKING: @@ -65,13 +65,14 @@ class BadSpider(scrapy.Spider): def test_run_fail_spider(self, tmp_path: Path) -> None: ret, _, _ = self.runspider( - tmp_path, "import scrapy\n" + inspect.getsource(ExceptionSpider) + tmp_path, "from scrapy import Spider\n" + inspect.getsource(ExceptionSpider) ) assert ret != 0 def test_run_good_spider(self, tmp_path: Path) -> None: ret, _, _ = self.runspider( - tmp_path, "import scrapy\n" + inspect.getsource(NoRequestsSpider) + tmp_path, + "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider), ) assert ret == 0 diff --git a/tests/test_commands.py b/tests/test_commands.py index bb4d96f00..9e7d4d5a1 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -4,7 +4,6 @@ import argparse import json import sys from io import StringIO -from shutil import copytree from typing import TYPE_CHECKING from unittest import mock @@ -16,7 +15,8 @@ from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings from scrapy.utils.reactor import _asyncio_reactor_path -from tests.utils.cmdline import call, proc +from tests.utils.base_commands import TestProjectBase +from tests.utils.cmdline import call, proc, write_recording_editor if TYPE_CHECKING: from pathlib import Path @@ -109,29 +109,6 @@ class TestCommandSettings: ) -class TestProjectBase: - """A base class for tests that may need a Scrapy project.""" - - project_name = "testproject" - - @pytest.fixture(scope="session") - def _proj_path_cached(self, tmp_path_factory: pytest.TempPathFactory) -> Path: - """Create a Scrapy project in a temporary directory and return its path. - - Used as a cache for ``proj_path``. - """ - tmp_path = tmp_path_factory.mktemp("proj") - call("startproject", self.project_name, cwd=tmp_path) - return tmp_path / self.project_name - - @pytest.fixture - def proj_path(self, tmp_path: Path, _proj_path_cached: Path) -> Path: - """Copy a pre-generated Scrapy project into a temporary directory and return its path.""" - proj_path = tmp_path / self.project_name - copytree(_proj_path_cached, proj_path) - return proj_path - - class TestCommandCrawlerProcess(TestProjectBase): """Test that the command uses the expected kind of *CrawlerProcess and produces expected errors when needed.""" @@ -430,9 +407,7 @@ class TestEditCommand(TestProjectBase): spider = proj_path / self.project_name / "spiders" / "example.py" edited = proj_path / "edited.txt" editor = proj_path / "fake-editor.sh" - # Records the file it is asked to open ($2) into the file given as $1. - editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8") - editor.chmod(0o755) + write_recording_editor(editor) monkeypatch.setenv("EDITOR", f"{editor} {edited}") assert call("genspider", "example", "example.com", cwd=proj_path) == 0 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 8c1716345..9c479068a 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -14,14 +14,13 @@ from twisted.internet.ssl import Certificate from twisted.python.failure import Failure from scrapy import Spider, signals -from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload from scrapy.http import Request from scrapy.http.response import Response -from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.engine import format_engine_status, get_engine_status from scrapy.utils.python import to_unicode -from scrapy.utils.test import get_crawler, get_reactor_settings +from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE from tests.spiders import ( AsyncDefAsyncioGenComplexSpider, @@ -430,50 +429,6 @@ with multiples lines ) assert not crawler.crawling - @coroutine_test - async def test_crawlerrunner_accepts_crawler( - self, caplog: pytest.LogCaptureFixture, mockserver: MockServer - ) -> None: - crawler = Crawler(SimpleSpider, get_reactor_settings()) - runner = CrawlerRunner() - with caplog.at_level(logging.DEBUG): - await maybe_deferred_to_future( - runner.crawl( - crawler, - mockserver.url("/status?n=200"), - mockserver=mockserver, - ) - ) - assert "Got response 200" in caplog.text - - @coroutine_test - async def test_crawl_multiple( - self, caplog: pytest.LogCaptureFixture, mockserver: MockServer - ) -> None: - settings_dict = get_reactor_settings() - runner_cls = ( - CrawlerRunner - if settings_dict.get("TWISTED_REACTOR_ENABLED", True) - else AsyncCrawlerRunner - ) - runner = runner_cls(settings_dict) - runner.crawl( - SimpleSpider, - mockserver.url("/status?n=200"), - mockserver=mockserver, - ) - runner.crawl( - SimpleSpider, - mockserver.url("/status?n=503"), - mockserver=mockserver, - ) - - with caplog.at_level(logging.DEBUG): - await ensure_awaitable(runner.join()) - - self._assert_retried(caplog.text) - assert "Got response 200" in caplog.text - @coroutine_test async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None: crawler = get_crawler(SimpleSpider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6a1eea027..b4f906e25 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -10,22 +10,14 @@ from typing import TYPE_CHECKING, Any, ClassVar from unittest.mock import MagicMock import pytest -from zope.interface.exceptions import MultipleInvalid import scrapy from scrapy import Spider -from scrapy.crawler import ( - AsyncCrawlerProcess, - AsyncCrawlerRunner, - Crawler, - CrawlerProcess, - CrawlerRunner, - CrawlerRunnerBase, -) +from scrapy.crawler import AsyncCrawlerProcess, Crawler, CrawlerProcess from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings -from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.log import ( _uninstall_scrapy_root_handler, configure_logging, @@ -33,6 +25,8 @@ from scrapy.utils.log import ( ) from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings +from tests.spiders import NoRequestsSpider +from tests.utils import assert_option_is_default from tests.utils.decorators import coroutine_test if TYPE_CHECKING: @@ -52,14 +46,7 @@ def get_raw_crawler( return Crawler(spidercls or DefaultSpider, settings) -class TestBaseCrawler: - @staticmethod - def assertOptionIsDefault(settings: Settings, key: str) -> None: - assert isinstance(settings, Settings) - assert settings[key] == getattr(default_settings, key) - - -class TestCrawler(TestBaseCrawler): +class TestCrawler: def test_populate_spidercls_settings(self) -> None: spider_settings: dict[str, Any] = { "TEST1": "spider", @@ -90,11 +77,11 @@ class TestCrawler(TestBaseCrawler): def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) assert crawler.settings["foo"] == "bar" - self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") + assert_option_is_default(crawler.settings, "RETRY_ENABLED") def test_crawler_accepts_None(self) -> None: crawler = Crawler(DefaultSpider) - self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") + assert_option_is_default(crawler.settings, "RETRY_ENABLED") def test_crawler_rejects_spider_objects(self) -> None: with pytest.raises(ValueError, match="spidercls argument must be a class"): @@ -586,78 +573,6 @@ class TestCrawlerLogging: assert "debug message" in logged -class SpiderLoaderWithWrongInterface: - def unneeded_method(self) -> None: - pass - - -class TestCrawlerRunner(TestBaseCrawler): - def test_spider_manager_verify_interface(self) -> None: - settings = Settings( - { - "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, - } - ) - with pytest.raises(MultipleInvalid): - CrawlerRunner(settings) - - def test_crawler_runner_accepts_dict(self) -> None: - runner = CrawlerRunner({"foo": "bar"}) - assert runner.settings["foo"] == "bar" - self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - - def test_crawler_runner_accepts_None(self) -> None: - runner = CrawlerRunner() - self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - - -class TestAsyncCrawlerRunner(TestBaseCrawler): - def test_spider_manager_verify_interface(self) -> None: - settings = Settings( - { - "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, - } - ) - with pytest.raises(MultipleInvalid): - AsyncCrawlerRunner(settings) - - def test_crawler_runner_accepts_dict(self) -> None: - runner = AsyncCrawlerRunner({"foo": "bar"}) - assert runner.settings["foo"] == "bar" - self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - - def test_crawler_runner_accepts_None(self) -> None: - runner = AsyncCrawlerRunner() - self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - - -class TestCrawlerProcess(TestBaseCrawler): - def test_crawler_process_accepts_dict(self) -> None: - runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False) - assert runner.settings["foo"] == "bar" - self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - - def test_crawler_process_accepts_None(self) -> None: - runner = CrawlerProcess(install_root_handler=False) - self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - - -@pytest.mark.only_asyncio -class TestAsyncCrawlerProcess(TestBaseCrawler): - def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None: - runner = AsyncCrawlerProcess( - {"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"}, - install_root_handler=False, - ) - assert runner.settings["foo"] == "bar" - self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - - @pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False - def test_crawler_process_accepts_None(self) -> None: - runner = AsyncCrawlerProcess(install_root_handler=False) - self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - - class TestAsyncCrawlerProcessReactorlessHelpers: """Unit tests for the reactorless shutdown helpers of AsyncCrawlerProcess. @@ -797,161 +712,6 @@ class TestAsyncCrawlerProcessReactorlessHelpers: ) -@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) -def test_runner_settings_applied_to_crawler_instance( - runner_cls: type[CrawlerRunnerBase], -) -> None: - runner = runner_cls({"FOO": "runner"}) - crawler = Crawler(DefaultSpider) - result = runner.create_crawler(crawler) - assert result is crawler - assert result.settings["FOO"] == "runner" - - -@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) -def test_spider_custom_settings_override_runner( - runner_cls: type[CrawlerRunnerBase], -) -> None: - class MySpider(DefaultSpider): - custom_settings = {"FOO": "spider"} - - runner = runner_cls({"FOO": "runner"}) - crawler = Crawler(MySpider) - runner.create_crawler(crawler) - assert crawler.settings["FOO"] == "spider" - - -def test_create_crawler_instance_consistent_with_spider_class() -> None: - runner = AsyncCrawlerRunner({"FOO": "runner"}) - - crawler_from_class = runner.create_crawler(DefaultSpider) - - pre_built = Crawler(DefaultSpider) - runner.create_crawler(pre_built) - - assert crawler_from_class.settings["FOO"] == "runner" - assert pre_built.settings["FOO"] == "runner" - - -@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) -def test_create_crawler_rejects_spider_object( - runner_cls: type[CrawlerRunnerBase], -) -> None: - runner = runner_cls() - with pytest.raises(ValueError, match="cannot be a spider object"): - runner.create_crawler(DefaultSpider()) # type: ignore[arg-type] - - -@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) -def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None: - runner = runner_cls() - with pytest.raises(ValueError, match="cannot be a spider object"): - runner.crawl(DefaultSpider()) # type: ignore[arg-type] - - -class ExceptionSpider(scrapy.Spider): - name = "exception" - - @classmethod - def from_crawler(cls, crawler, *args, **kwargs): - raise ValueError("Exception in from_crawler method") - - -class NoRequestsSpider(scrapy.Spider): - name = "no_request" - - async def start(self): - return - yield - - -@pytest.mark.requires_reactor # CrawlerRunner requires a reactor -class TestCrawlerRunnerHasSpider: - @pytest.fixture - def runner(self) -> CrawlerRunnerBase: - return CrawlerRunner(get_reactor_settings()) - - @staticmethod - async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None: - await ensure_awaitable(runner.crawl(spider)) - - @coroutine_test - async def test_crawler_runner_bootstrap_successful( - self, runner: CrawlerRunnerBase - ) -> None: - await self._crawl(runner, NoRequestsSpider) - assert not runner.bootstrap_failed - - @coroutine_test - async def test_crawler_runner_bootstrap_successful_for_several( - self, runner: CrawlerRunnerBase - ) -> None: - await self._crawl(runner, NoRequestsSpider) - await self._crawl(runner, NoRequestsSpider) - assert not runner.bootstrap_failed - - @coroutine_test - async def test_crawler_runner_bootstrap_failed( - self, runner: CrawlerRunnerBase - ) -> None: - try: - await self._crawl(runner, ExceptionSpider) - except ValueError: - pass - else: - pytest.fail("Exception should be raised from spider") - - assert runner.bootstrap_failed - - @coroutine_test - async def test_crawler_runner_bootstrap_failed_for_several( - self, runner: CrawlerRunnerBase - ) -> None: - try: - await self._crawl(runner, ExceptionSpider) - except ValueError: - pass - else: - pytest.fail("Exception should be raised from spider") - - await self._crawl(runner, NoRequestsSpider) - - assert runner.bootstrap_failed - - @coroutine_test - async def test_crawler_runner_asyncio_enabled_true( - self, reactor_pytest: str - ) -> None: - if reactor_pytest != "asyncio": - runner = CrawlerRunner( - settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - } - ) - with pytest.raises( - Exception, - match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", - ): - await self._crawl(runner, NoRequestsSpider) - else: - runner = CrawlerRunner( - settings={ - "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", - } - ) - await self._crawl(runner, NoRequestsSpider) - - -@pytest.mark.only_asyncio -class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): - @pytest.fixture - def runner(self) -> CrawlerRunnerBase: - return AsyncCrawlerRunner(get_reactor_settings()) - - def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override] - pytest.skip("This test is only for CrawlerRunner") - - @pytest.mark.parametrize( ("settings", "items"), [ diff --git a/tests/test_crawler_runners.py b/tests/test_crawler_runners.py new file mode 100644 index 000000000..2f3649b43 --- /dev/null +++ b/tests/test_crawler_runners.py @@ -0,0 +1,284 @@ +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +import pytest +from zope.interface.exceptions import MultipleInvalid + +from scrapy.crawler import ( + AsyncCrawlerProcess, + AsyncCrawlerRunner, + Crawler, + CrawlerProcess, + CrawlerRunner, + CrawlerRunnerBase, +) +from scrapy.settings import Settings +from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_reactor_settings +from tests.spiders import ExceptionSpider, NoRequestsSpider, SimpleSpider +from tests.utils import assert_option_is_default +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from scrapy import Spider + from tests.mockserver.http import MockServer + + +class SpiderLoaderWithWrongInterface: + def unneeded_method(self) -> None: + pass + + +class TestCrawlerRunner: + def test_spider_manager_verify_interface(self) -> None: + settings = Settings( + { + "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, + } + ) + with pytest.raises(MultipleInvalid): + CrawlerRunner(settings) + + def test_crawler_runner_accepts_dict(self) -> None: + runner = CrawlerRunner({"foo": "bar"}) + assert runner.settings["foo"] == "bar" + assert_option_is_default(runner.settings, "RETRY_ENABLED") + + def test_crawler_runner_accepts_None(self) -> None: + runner = CrawlerRunner() + assert_option_is_default(runner.settings, "RETRY_ENABLED") + + +class TestAsyncCrawlerRunner: + def test_spider_manager_verify_interface(self) -> None: + settings = Settings( + { + "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, + } + ) + with pytest.raises(MultipleInvalid): + AsyncCrawlerRunner(settings) + + def test_crawler_runner_accepts_dict(self) -> None: + runner = AsyncCrawlerRunner({"foo": "bar"}) + assert runner.settings["foo"] == "bar" + assert_option_is_default(runner.settings, "RETRY_ENABLED") + + def test_crawler_runner_accepts_None(self) -> None: + runner = AsyncCrawlerRunner() + assert_option_is_default(runner.settings, "RETRY_ENABLED") + + +class TestCrawlerProcess: + def test_crawler_process_accepts_dict(self) -> None: + runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False) + assert runner.settings["foo"] == "bar" + assert_option_is_default(runner.settings, "RETRY_ENABLED") + + def test_crawler_process_accepts_None(self) -> None: + runner = CrawlerProcess(install_root_handler=False) + assert_option_is_default(runner.settings, "RETRY_ENABLED") + + +@pytest.mark.only_asyncio +class TestAsyncCrawlerProcess: + def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None: + runner = AsyncCrawlerProcess( + {"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"}, + install_root_handler=False, + ) + assert runner.settings["foo"] == "bar" + assert_option_is_default(runner.settings, "RETRY_ENABLED") + + @pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False + def test_crawler_process_accepts_None(self) -> None: + runner = AsyncCrawlerProcess(install_root_handler=False) + assert_option_is_default(runner.settings, "RETRY_ENABLED") + + +@pytest.mark.requires_reactor # CrawlerRunner requires a reactor +class TestCrawlerRunnerHasSpider: + @pytest.fixture + def runner(self) -> CrawlerRunnerBase: + return CrawlerRunner(get_reactor_settings()) + + @staticmethod + async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None: + await ensure_awaitable(runner.crawl(spider)) + + @coroutine_test + async def test_crawler_runner_bootstrap_successful( + self, runner: CrawlerRunnerBase + ) -> None: + await self._crawl(runner, NoRequestsSpider) + assert not runner.bootstrap_failed + + @coroutine_test + async def test_crawler_runner_bootstrap_successful_for_several( + self, runner: CrawlerRunnerBase + ) -> None: + await self._crawl(runner, NoRequestsSpider) + await self._crawl(runner, NoRequestsSpider) + assert not runner.bootstrap_failed + + @coroutine_test + async def test_crawler_runner_bootstrap_failed( + self, runner: CrawlerRunnerBase + ) -> None: + try: + await self._crawl(runner, ExceptionSpider) + except ValueError: + pass + else: + pytest.fail("Exception should be raised from spider") + + assert runner.bootstrap_failed + + @coroutine_test + async def test_crawler_runner_bootstrap_failed_for_several( + self, runner: CrawlerRunnerBase + ) -> None: + try: + await self._crawl(runner, ExceptionSpider) + except ValueError: + pass + else: + pytest.fail("Exception should be raised from spider") + + await self._crawl(runner, NoRequestsSpider) + + assert runner.bootstrap_failed + + @coroutine_test + async def test_crawler_runner_asyncio_enabled_true( + self, reactor_pytest: str + ) -> None: + if reactor_pytest != "asyncio": + runner = CrawlerRunner( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + ) + with pytest.raises( + Exception, + match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", + ): + await self._crawl(runner, NoRequestsSpider) + else: + runner = CrawlerRunner( + settings={ + "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + } + ) + await self._crawl(runner, NoRequestsSpider) + + +@pytest.mark.only_asyncio +class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): + @pytest.fixture + def runner(self) -> CrawlerRunnerBase: + return AsyncCrawlerRunner(get_reactor_settings()) + + def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override] + pytest.skip("This test is only for CrawlerRunner") + + +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_runner_settings_applied_to_crawler_instance( + runner_cls: type[CrawlerRunnerBase], +) -> None: + runner = runner_cls({"FOO": "runner"}) + crawler = Crawler(DefaultSpider) + result = runner.create_crawler(crawler) + assert result is crawler + assert result.settings["FOO"] == "runner" + + +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_spider_custom_settings_override_runner( + runner_cls: type[CrawlerRunnerBase], +) -> None: + class MySpider(DefaultSpider): + custom_settings = {"FOO": "spider"} + + runner = runner_cls({"FOO": "runner"}) + crawler = Crawler(MySpider) + runner.create_crawler(crawler) + assert crawler.settings["FOO"] == "spider" + + +def test_create_crawler_instance_consistent_with_spider_class() -> None: + runner = AsyncCrawlerRunner({"FOO": "runner"}) + + crawler_from_class = runner.create_crawler(DefaultSpider) + + pre_built = Crawler(DefaultSpider) + runner.create_crawler(pre_built) + + assert crawler_from_class.settings["FOO"] == "runner" + assert pre_built.settings["FOO"] == "runner" + + +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_create_crawler_rejects_spider_object( + runner_cls: type[CrawlerRunnerBase], +) -> None: + runner = runner_cls() + with pytest.raises(ValueError, match="cannot be a spider object"): + runner.create_crawler(DefaultSpider()) # type: ignore[arg-type] + + +@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner]) +def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None: + runner = runner_cls() + with pytest.raises(ValueError, match="cannot be a spider object"): + runner.crawl(DefaultSpider()) # type: ignore[arg-type] + + +@coroutine_test +async def test_crawlerrunner_accepts_crawler( + caplog: pytest.LogCaptureFixture, mockserver: MockServer +) -> None: + crawler = Crawler(SimpleSpider, get_reactor_settings()) + runner = CrawlerRunner() + with caplog.at_level(logging.DEBUG): + await maybe_deferred_to_future( + runner.crawl( + crawler, + mockserver.url("/status?n=200"), + mockserver=mockserver, + ) + ) + assert "Got response 200" in caplog.text + + +@coroutine_test +async def test_crawl_multiple( + caplog: pytest.LogCaptureFixture, mockserver: MockServer +) -> None: + settings_dict = get_reactor_settings() + runner_cls = ( + CrawlerRunner + if settings_dict.get("TWISTED_REACTOR_ENABLED", True) + else AsyncCrawlerRunner + ) + runner = runner_cls(settings_dict) + runner.crawl( + SimpleSpider, + mockserver.url("/status?n=200"), + mockserver=mockserver, + ) + runner.crawl( + SimpleSpider, + mockserver.url("/status?n=503"), + mockserver=mockserver, + ) + + with caplog.at_level(logging.DEBUG): + await ensure_awaitable(runner.join()) + + assert "Got response 200" in caplog.text + assert "Gave up retrying" in caplog.text diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index 1da7bbf3e..0824ab531 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -16,11 +16,8 @@ from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests.test_downloadermiddleware_redirect_base import ( - REDIRECT_SCHEME_CASES, - SCHEME_PARAMS, - Base, -) +from tests.test_downloadermiddleware_redirect_base import Base +from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMS class TestRedirectMiddleware(Base.Test): diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.py index 32935769f..bd8bc796c 100644 --- a/tests/test_downloadermiddleware_redirect_base.py +++ b/tests/test_downloadermiddleware_redirect_base.py @@ -1,7 +1,5 @@ from __future__ import annotations -from itertools import product - import pytest from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware @@ -10,43 +8,6 @@ from scrapy.http import Request, Response from scrapy.utils.misc import set_environ from scrapy.utils.test import get_crawler -SCHEME_PARAMS = ("url", "location", "target") -HTTP_SCHEMES = ("http", "https") -NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo") -REDIRECT_SCHEME_CASES = ( - # http/https → http/https redirects - *( - ( - f"{input_scheme}://example.com/a", - f"{output_scheme}://example.com/b", - f"{output_scheme}://example.com/b", - ) - for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2) - ), - # http/https → data/file/ftp/s3/foo does not redirect - *( - ( - f"{input_scheme}://example.com/a", - f"{output_scheme}://example.com/b", - None, - ) - for input_scheme in HTTP_SCHEMES - for output_scheme in NON_HTTP_SCHEMES - ), - # http/https → relative redirects - *( - ( - f"{scheme}://example.com/a", - location, - f"{scheme}://example.com/b", - ) - for scheme in HTTP_SCHEMES - for location in ("//example.com/b", "/b") - ), - # Note: We do not test data/file/ftp/s3 schemes for the initial URL - # because their download handlers cannot return a status code of 3xx. -) - class Base: class Test: diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py index d849cc8fb..b5d39080c 100644 --- a/tests/test_downloadermiddleware_redirect_metarefresh.py +++ b/tests/test_downloadermiddleware_redirect_metarefresh.py @@ -12,12 +12,12 @@ from scrapy.http import HtmlResponse, Request, Response from scrapy.spiders import Spider from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler -from tests.test_downloadermiddleware_redirect_base import ( +from tests.test_downloadermiddleware_redirect_base import Base +from tests.utils.redirect import ( HTTP_SCHEMES, NON_HTTP_SCHEMES, REDIRECT_SCHEME_CASES, SCHEME_PARAMS, - Base, ) diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 082fc743e..f82041a62 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -15,8 +15,8 @@ from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings from scrapy.utils.asyncio import call_later from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future -from tests.test_robotstxt_interface import rerp_available from tests.utils.decorators import coroutine_test +from tests.utils.robotstxt import rerp_available if TYPE_CHECKING: from scrapy.crawler import Crawler diff --git a/tests/test_engine.py b/tests/test_engine.py index 2b857d35a..3ab8f6d1b 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -6,8 +6,8 @@ import subprocess import sys from collections import defaultdict from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, cast -from unittest.mock import Mock, call +from typing import TYPE_CHECKING, Any +from unittest.mock import Mock from urllib.parse import urlparse import attr @@ -25,7 +25,6 @@ from scrapy.http import Headers, Request, Response from scrapy.item import Field, Item from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider -from scrapy.statscollectors import MemoryStatsCollector from scrapy.utils.defer import ( _schedule_coro, deferred_from_coro, @@ -42,8 +41,6 @@ if TYPE_CHECKING: from twisted.python.failure import Failure - from scrapy.core.scheduler import Scheduler - from scrapy.crawler import Crawler from tests.mockserver.http import MockServer @@ -515,98 +512,6 @@ class TestEngine(TestEngineBase): assert "AssertionError" not in stderr_str, stderr_str -class TestEngineDownloadAsync: - """Test cases for ExecutionEngine.download_async().""" - - @pytest.fixture - def engine(self) -> ExecutionEngine: - crawler = get_crawler(MySpider) - engine = ExecutionEngine(crawler, lambda _: None) - engine.downloader.close() - engine.downloader = Mock() - engine._slot = Mock() - engine._slot.inprogress = set() - return engine - - @staticmethod - async def _download(engine: ExecutionEngine, request: Request) -> Response: - return await engine.download_async(request) - - @coroutine_test - async def test_download_async_success(self, engine): - """Test basic successful async download of a request.""" - request = Request("http://example.com") - response = Response("http://example.com", body=b"test body") - engine.spider = Mock() - engine.downloader.fetch.return_value = defer.succeed(response) - engine._slot.add_request = Mock() - engine._slot.remove_request = Mock() - - result = await self._download(engine, request) - assert result == response - engine._slot.add_request.assert_called_once_with(request) - engine._slot.remove_request.assert_called_once_with(request) - engine.downloader.fetch.assert_called_once_with(request) - - @coroutine_test - async def test_download_async_redirect(self, engine): - """Test async download with a redirect request.""" - original_request = Request("http://example.com") - redirect_request = Request("http://example.com/redirect") - final_response = Response("http://example.com/redirect", body=b"redirected") - - # First call returns redirect request, second call returns final response - engine.downloader.fetch.side_effect = [ - defer.succeed(redirect_request), - defer.succeed(final_response), - ] - engine.spider = Mock() - engine._slot.add_request = Mock() - engine._slot.remove_request = Mock() - - result = await self._download(engine, original_request) - assert result == final_response - assert engine.downloader.fetch.call_count == 2 - engine._slot.add_request.assert_has_calls( - [call(original_request), call(redirect_request)] - ) - engine._slot.remove_request.assert_has_calls( - [call(original_request), call(redirect_request)] - ) - - @coroutine_test - async def test_download_async_no_spider(self, engine): - """Test async download attempt when no spider is available.""" - request = Request("http://example.com") - engine.spider = None - with pytest.raises(RuntimeError, match="No open spider to crawl:"): - await self._download(engine, request) - - @coroutine_test - async def test_download_async_failure(self, engine): - """Test async download when the downloader raises an exception.""" - request = Request("http://example.com") - error = RuntimeError("Download failed") - engine.spider = Mock() - engine.downloader.fetch.return_value = defer.fail(error) - engine._slot.add_request = Mock() - engine._slot.remove_request = Mock() - - with pytest.raises(RuntimeError, match="Download failed"): - await self._download(engine, request) - engine._slot.add_request.assert_called_once_with(request) - engine._slot.remove_request.assert_called_once_with(request) - - -@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") -class TestEngineDownload(TestEngineDownloadAsync): - """Test cases for ExecutionEngine.download().""" - - @staticmethod - async def _download(engine: ExecutionEngine, request: Request) -> Response: - return await maybe_deferred_to_future(engine.download(request)) - - @coroutine_test async def test_request_scheduled_signal(): class TestScheduler(BaseScheduler): @@ -640,132 +545,3 @@ async def test_request_scheduled_signal(): f"{scheduler.enqueued!r} != [{keep_request!r}]" ) crawler.signals.disconnect(signal_handler, signals.request_scheduled) - - -class TestEngineCloseSpider: - """Tests for exception handling coverage during close_spider_async().""" - - @pytest.fixture - def crawler(self) -> Crawler: - crawler = get_crawler(DefaultSpider) - crawler.spider = crawler._create_spider() - return crawler - - @coroutine_test - async def test_no_slot(self, crawler: Crawler) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - slot = engine._slot - engine._slot = None - with pytest.raises(RuntimeError, match="Engine slot not assigned"): - await engine.close_spider_async() - # close it correctly - engine._slot = slot - await engine.close_spider_async() - - @coroutine_test - async def test_no_spider(self, crawler: Crawler) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - with pytest.raises(RuntimeError, match="Spider not opened"): - await engine.close_spider_async() - engine.downloader.close() # cleanup - - @coroutine_test - async def test_exception_slot( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - assert engine._slot - del engine._slot.heartbeat - await engine.close_spider_async() - assert "Slot close failure" in caplog.text - - @coroutine_test - async def test_exception_downloader( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - del engine.downloader.slots - await engine.close_spider_async() - assert "Downloader close failure" in caplog.text - - @coroutine_test - async def test_exception_scraper( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - engine.scraper.slot = None - await engine.close_spider_async() - assert "Scraper close failure" in caplog.text - - @coroutine_test - async def test_exception_scheduler( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - assert engine._slot - del cast("Scheduler", engine._slot.scheduler).dqs - await engine.close_spider_async() - assert "Scheduler close failure" in caplog.text - - @coroutine_test - async def test_exception_signal( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - signal_manager = engine.signals - del engine.signals - await engine.close_spider_async() - assert "Error while sending spider_close signal" in caplog.text - # send the spider_closed signal to close various components - await signal_manager.send_catch_log_async( - signal=signals.spider_closed, - spider=engine.spider, - reason="cancelled", - ) - - @coroutine_test - async def test_exception_stats( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: None) - crawler.engine = engine - await engine.open_spider_async() - assert isinstance(crawler.stats, MemoryStatsCollector) - del crawler.stats.spider_stats - await engine.close_spider_async() - assert "Stats close failure" in caplog.text - - @coroutine_test - async def test_exception_callback( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError())) - crawler.engine = engine - await engine.open_spider_async() - await engine.close_spider_async() - assert "Error running spider_closed_callback" in caplog.text - - @coroutine_test - async def test_exception_async_callback( - self, crawler: Crawler, caplog: pytest.LogCaptureFixture - ) -> None: - async def cb(_): - raise ValueError - - engine = ExecutionEngine(crawler, cb) - crawler.engine = engine - await engine.open_spider_async() - await engine.close_spider_async() - assert "Error running spider_closed_callback" in caplog.text diff --git a/tests/test_engine_close_spider.py b/tests/test_engine_close_spider.py new file mode 100644 index 000000000..8e9c9bd43 --- /dev/null +++ b/tests/test_engine_close_spider.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, cast + +import pytest +from twisted.internet import defer + +from scrapy import signals +from scrapy.core.engine import ExecutionEngine +from scrapy.statscollectors import MemoryStatsCollector +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from scrapy.core.scheduler import Scheduler + from scrapy.crawler import Crawler + + +@pytest.fixture +def crawler() -> Crawler: + crawler = get_crawler(DefaultSpider) + crawler.spider = crawler._create_spider() + return crawler + + +@coroutine_test +async def test_no_slot(crawler: Crawler) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + slot = engine._slot + engine._slot = None + with pytest.raises(RuntimeError, match="Engine slot not assigned"): + await engine.close_spider_async() + # close it correctly + engine._slot = slot + await engine.close_spider_async() + + +@coroutine_test +async def test_no_spider(crawler: Crawler) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + with pytest.raises(RuntimeError, match="Spider not opened"): + await engine.close_spider_async() + engine.downloader.close() # cleanup + + +@coroutine_test +async def test_exception_slot( + crawler: Crawler, caplog: pytest.LogCaptureFixture +) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + assert engine._slot + del engine._slot.heartbeat + await engine.close_spider_async() + assert "Slot close failure" in caplog.text + + +@coroutine_test +async def test_exception_downloader( + crawler: Crawler, caplog: pytest.LogCaptureFixture +) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + del engine.downloader.slots + await engine.close_spider_async() + assert "Downloader close failure" in caplog.text + + +@coroutine_test +async def test_exception_scraper( + crawler: Crawler, caplog: pytest.LogCaptureFixture +) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + engine.scraper.slot = None + await engine.close_spider_async() + assert "Scraper close failure" in caplog.text + + +@coroutine_test +async def test_exception_scheduler( + crawler: Crawler, caplog: pytest.LogCaptureFixture +) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + assert engine._slot + del cast("Scheduler", engine._slot.scheduler).dqs + await engine.close_spider_async() + assert "Scheduler close failure" in caplog.text + + +@coroutine_test +async def test_exception_signal( + crawler: Crawler, caplog: pytest.LogCaptureFixture +) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + signal_manager = engine.signals + del engine.signals + await engine.close_spider_async() + assert "Error while sending spider_close signal" in caplog.text + # send the spider_closed signal to close various components + await signal_manager.send_catch_log_async( + signal=signals.spider_closed, + spider=engine.spider, + reason="cancelled", + ) + + +@coroutine_test +async def test_exception_stats( + crawler: Crawler, caplog: pytest.LogCaptureFixture +) -> None: + engine = ExecutionEngine(crawler, lambda _: None) + crawler.engine = engine + await engine.open_spider_async() + assert isinstance(crawler.stats, MemoryStatsCollector) + del crawler.stats.spider_stats + await engine.close_spider_async() + assert "Stats close failure" in caplog.text + + +@coroutine_test +async def test_exception_callback( + crawler: Crawler, caplog: pytest.LogCaptureFixture +) -> None: + engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError())) + crawler.engine = engine + await engine.open_spider_async() + await engine.close_spider_async() + assert "Error running spider_closed_callback" in caplog.text + + +@coroutine_test +async def test_exception_async_callback( + crawler: Crawler, caplog: pytest.LogCaptureFixture +) -> None: + async def cb(_): + raise ValueError + + engine = ExecutionEngine(crawler, cb) + crawler.engine = engine + await engine.open_spider_async() + await engine.close_spider_async() + assert "Error running spider_closed_callback" in caplog.text diff --git a/tests/test_engine_download.py b/tests/test_engine_download.py new file mode 100644 index 000000000..f15bfd5e2 --- /dev/null +++ b/tests/test_engine_download.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +from unittest.mock import Mock, call + +import pytest +from twisted.internet import defer + +from scrapy.core.engine import ExecutionEngine +from scrapy.http import Request, Response +from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + + +@pytest.fixture +def engine() -> ExecutionEngine: + crawler = get_crawler(DefaultSpider) + engine = ExecutionEngine(crawler, lambda _: None) + engine.downloader.close() + engine.downloader = Mock() + engine._slot = Mock() + engine._slot.inprogress = set() + return engine + + +class TestEngineDownloadAsync: + """Test cases for ExecutionEngine.download_async().""" + + @staticmethod + async def _download(engine: ExecutionEngine, request: Request) -> Response: + return await engine.download_async(request) + + @coroutine_test + async def test_download_async_success(self, engine): + """Test basic successful async download of a request.""" + request = Request("http://example.com") + response = Response("http://example.com", body=b"test body") + engine.spider = Mock() + engine.downloader.fetch.return_value = defer.succeed(response) + engine._slot.add_request = Mock() + engine._slot.remove_request = Mock() + + result = await self._download(engine, request) + assert result == response + engine._slot.add_request.assert_called_once_with(request) + engine._slot.remove_request.assert_called_once_with(request) + engine.downloader.fetch.assert_called_once_with(request) + + @coroutine_test + async def test_download_async_redirect(self, engine): + """Test async download with a redirect request.""" + original_request = Request("http://example.com") + redirect_request = Request("http://example.com/redirect") + final_response = Response("http://example.com/redirect", body=b"redirected") + + # First call returns redirect request, second call returns final response + engine.downloader.fetch.side_effect = [ + defer.succeed(redirect_request), + defer.succeed(final_response), + ] + engine.spider = Mock() + engine._slot.add_request = Mock() + engine._slot.remove_request = Mock() + + result = await self._download(engine, original_request) + assert result == final_response + assert engine.downloader.fetch.call_count == 2 + engine._slot.add_request.assert_has_calls( + [call(original_request), call(redirect_request)] + ) + engine._slot.remove_request.assert_has_calls( + [call(original_request), call(redirect_request)] + ) + + @coroutine_test + async def test_download_async_no_spider(self, engine): + """Test async download attempt when no spider is available.""" + request = Request("http://example.com") + engine.spider = None + with pytest.raises(RuntimeError, match="No open spider to crawl:"): + await self._download(engine, request) + + @coroutine_test + async def test_download_async_failure(self, engine): + """Test async download when the downloader raises an exception.""" + request = Request("http://example.com") + error = RuntimeError("Download failed") + engine.spider = Mock() + engine.downloader.fetch.return_value = defer.fail(error) + engine._slot.add_request = Mock() + engine._slot.remove_request = Mock() + + with pytest.raises(RuntimeError, match="Download failed"): + await self._download(engine, request) + engine._slot.add_request.assert_called_once_with(request) + engine._slot.remove_request.assert_called_once_with(request) + + +@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") +class TestEngineDownload(TestEngineDownloadAsync): + """Test cases for ExecutionEngine.download().""" + + @staticmethod + async def _download(engine: ExecutionEngine, request: Request) -> Response: + return await maybe_deferred_to_future(engine.download(request)) diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 6cc8c0650..ca52961f4 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -2,13 +2,13 @@ from __future__ import annotations from collections import deque from logging import ERROR -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from scrapy import Request, Spider, signals +from scrapy.core.scheduler import BaseScheduler from scrapy.utils.asyncio import call_later from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer -from tests.test_scheduler import MemoryScheduler from tests.utils import async_sleep from tests.utils.decorators import coroutine_test @@ -18,6 +18,38 @@ if TYPE_CHECKING: from scrapy.http import Response +class MemoryScheduler(BaseScheduler): + paused = False + + def __init__(self, *args: Any, **kwargs: Any): + super().__init__(*args, **kwargs) + self.queue: deque[Request] = deque( + Request(value) if isinstance(value, str) else value + for value in getattr(self, "queue", []) + ) + + def enqueue_request(self, request: Request) -> bool: + self.queue.append(request) + return True + + def has_pending_requests(self) -> bool: + return self.paused or bool(self.queue) + + def next_request(self) -> Request | None: + if self.paused: + return None + try: + return self.queue.pop() + except IndexError: + return None + + def pause(self) -> None: + self.paused = True + + def unpause(self) -> None: + self.paused = False + + class TestMain: @coroutine_test async def test_sleep(self): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 5ab1a54c1..853243b97 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -13,8 +13,6 @@ from pathlib import Path from string import ascii_letters, digits from typing import IO, TYPE_CHECKING, Any from unittest import mock -from urllib.parse import urljoin -from urllib.request import pathname2url import lxml.etree import pytest @@ -39,19 +37,12 @@ from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import ItemSpider from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.feedexport import MyItem, MyItem2, path_to_url, printf_escape if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Iterable -def path_to_url(path: str | Path) -> str: - return urljoin("file:", pathname2url(str(path))) - - -def printf_escape(s: str) -> str: - return s.replace("%", "%%") - - class FromCrawlerMixin: init_with_crawler = False @@ -114,15 +105,6 @@ class LogOnStoreFileStorage: class TestFeedExportBase(ABC): mockserver: MockServer - class MyItem(scrapy.Item): - foo = scrapy.Field() - egg = scrapy.Field() - baz = scrapy.Field() - - class MyItem2(scrapy.Item): - foo = scrapy.Field() - hello = scrapy.Field() - def _random_temp_filename(self, inter_dir="") -> Path: chars = [random.choice(ascii_letters + digits) for _ in range(15)] filename = "".join(chars) @@ -517,21 +499,21 @@ class TestFeedExport(TestFeedExportBase): async def test_export_items(self): # feed exporters use field names from Item items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), ] rows = [ {"egg": "spam1", "foo": "bar1", "baz": ""}, {"egg": "spam2", "foo": "bar2", "baz": "quux2"}, ] - header = self.MyItem.fields.keys() + header = MyItem.fields.keys() await self.assertExported(items, header, rows) @coroutine_test async def test_pathlib_uri_with_placeholders(self): feed_dir = Path(self.temp_dir, "pathlib_placeholders") feed_dir.mkdir() - items = [self.MyItem({"foo": "bar1", "egg": "spam1"})] + items = [MyItem({"foo": "bar1", "egg": "spam1"})] class TestSpider(scrapy.Spider): name = "testspider" @@ -561,7 +543,7 @@ class TestFeedExport(TestFeedExportBase): # so the resulting file name can be asserted exactly. feed_dir = Path(self.temp_dir, "pathlib_spaces_unicode") feed_dir.mkdir() - items = [self.MyItem({"foo": "bar1", "egg": "spam1"})] + items = [MyItem({"foo": "bar1", "egg": "spam1"})] class TestSpider(scrapy.Spider): name = "testspider" @@ -590,7 +572,7 @@ class TestFeedExport(TestFeedExportBase): # and #5794. feed_dir = Path(self.temp_dir, "dir with spaces") feed_dir.mkdir() - items = [self.MyItem({"foo": "bar1", "egg": "spam1"})] + items = [MyItem({"foo": "bar1", "egg": "spam1"})] class TestSpider(scrapy.Spider): name = "testspider" @@ -627,7 +609,7 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_start_finish_exporting_items(self): items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem({"foo": "bar1", "egg": "spam1"}), ] settings = { "FEEDS": { @@ -665,7 +647,7 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_start_finish_exporting_items_exception(self): items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem({"foo": "bar1", "egg": "spam1"}), ] settings = { "FEEDS": { @@ -743,15 +725,15 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_export_multiple_item_classes(self): items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem2({"hello": "world2", "foo": "bar2"}), - self.MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem2({"hello": "world2", "foo": "bar2"}), + MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}), {"hello": "world4", "egg": "spam4"}, ] # by default, Scrapy uses fields of the first Item for CSV and # all fields for JSON Lines - header = self.MyItem.fields.keys() + header = MyItem.fields.keys() rows_csv = [ {"egg": "spam1", "foo": "bar1", "baz": ""}, {"egg": "", "foo": "bar2", "baz": ""}, @@ -826,8 +808,8 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_export_based_on_item_classes(self): items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem2({"hello": "world2", "foo": "bar2"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem2({"hello": "world2", "foo": "bar2"}), {"hello": "world3", "egg": "spam3"}, ] @@ -849,15 +831,15 @@ class TestFeedExport(TestFeedExportBase): "FEEDS": { self._random_temp_filename(): { "format": "csv", - "item_classes": [self.MyItem], + "item_classes": [MyItem], }, self._random_temp_filename(): { "format": "json", - "item_classes": [self.MyItem2], + "item_classes": [MyItem2], }, self._random_temp_filename(): { "format": "jsonlines", - "item_classes": [self.MyItem, self.MyItem2], + "item_classes": [MyItem, MyItem2], }, self._random_temp_filename(): { "format": "xml", @@ -872,13 +854,11 @@ class TestFeedExport(TestFeedExportBase): @coroutine_test async def test_export_based_on_custom_filters(self): items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem2({"hello": "world2", "foo": "bar2"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem2({"hello": "world2", "foo": "bar2"}), {"hello": "world3", "egg": "spam3"}, ] - MyItem = self.MyItem - class CustomFilter1: def __init__(self, feed_options): pass @@ -918,7 +898,7 @@ class TestFeedExport(TestFeedExportBase): }, self._random_temp_filename(): { "format": "jsonlines", - "item_classes": [self.MyItem, self.MyItem2], + "item_classes": [MyItem, MyItem2], "item_filter": CustomFilter3, }, }, @@ -957,7 +937,7 @@ class TestFeedExport(TestFeedExportBase): # FEED_EXPORT_FIELDS option allows to order export fields # and to select a subset of fields to export, both for Items and dicts. - for item_cls in [self.MyItem, dict]: + for item_cls in [MyItem, dict]: items = [ item_cls({"foo": "bar1", "egg": "spam1"}), item_cls({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 469985288..6e0536110 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -24,6 +24,7 @@ from scrapy.utils.test import get_crawler from tests.spiders import ItemSpider from tests.test_feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.feedexport import MyItem if TYPE_CHECKING: from os import PathLike @@ -197,9 +198,9 @@ class TestBatchDeliveries(TestFeedExportBase): async def test_export_items(self): """Test partial deliveries in all supported formats""" items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + MyItem({"foo": "bar3", "baz": "quux3"}), ] rows = [ {"egg": "spam1", "foo": "bar1", "baz": ""}, @@ -207,7 +208,7 @@ class TestBatchDeliveries(TestFeedExportBase): {"foo": "bar3", "baz": "quux3", "egg": ""}, ] settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2} - header = self.MyItem.fields.keys() + header = MyItem.fields.keys() await self.assertExported(items, header, rows, settings=settings) def test_wrong_path(self): @@ -349,9 +350,9 @@ class TestBatchDeliveries(TestFeedExportBase): So %(batch_id)d replaced with the current id. """ items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + MyItem({"foo": "bar3", "baz": "quux3"}), ] settings = { "FEEDS": { @@ -387,9 +388,9 @@ class TestBatchDeliveries(TestFeedExportBase): def test_s3_export(self): bucket = "mybucket" items = [ - self.MyItem({"foo": "bar1", "egg": "spam1"}), - self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), - self.MyItem({"foo": "bar3", "baz": "quux3"}), + MyItem({"foo": "bar1", "egg": "spam1"}), + MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}), + MyItem({"foo": "bar3", "baz": "quux3"}), ] class CustomS3FeedStorage(S3FeedStorage): diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py index 6ebcab152..83d976e94 100644 --- a/tests/test_feedexport_postprocess.py +++ b/tests/test_feedexport_postprocess.py @@ -13,8 +13,9 @@ from typing import TYPE_CHECKING, Any import pytest from scrapy.utils.test import get_crawler -from tests.test_feedexport import TestFeedExportBase, path_to_url, printf_escape +from tests.test_feedexport import TestFeedExportBase from tests.utils.decorators import coroutine_test +from tests.utils.feedexport import path_to_url, printf_escape if TYPE_CHECKING: from scrapy import Spider diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py index 5249736f2..da94a4e95 100644 --- a/tests/test_robotstxt_interface.py +++ b/tests/test_robotstxt_interface.py @@ -7,17 +7,7 @@ from scrapy.robotstxt import ( decode_robotstxt, ) from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER - - -def rerp_available() -> bool: - # check if robotexclusionrulesparser is installed - try: - from robotexclusionrulesparser import ( # noqa: PLC0415 - RobotExclusionRulesParser, # noqa: F401 - ) - except ImportError: - return False - return True +from tests.utils.robotstxt import rerp_available class BaseRobotParserTest: diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 39ce87b65..4fb594444 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -2,7 +2,6 @@ from __future__ import annotations import warnings from abc import ABC, abstractmethod -from collections import deque from contextlib import AbstractAsyncContextManager, asynccontextmanager from typing import TYPE_CHECKING, Any, NamedTuple, cast from unittest.mock import Mock @@ -10,7 +9,7 @@ from unittest.mock import Mock import pytest from scrapy.core.downloader import Downloader -from scrapy.core.scheduler import BaseScheduler, Scheduler +from scrapy.core.scheduler import Scheduler from scrapy.crawler import Crawler from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request @@ -27,38 +26,6 @@ if TYPE_CHECKING: from pathlib import Path -class MemoryScheduler(BaseScheduler): - paused = False - - def __init__(self, *args: Any, **kwargs: Any): - super().__init__(*args, **kwargs) - self.queue: deque[Request] = deque( - Request(value) if isinstance(value, str) else value - for value in getattr(self, "queue", []) - ) - - def enqueue_request(self, request: Request) -> bool: - self.queue.append(request) - return True - - def has_pending_requests(self) -> bool: - return self.paused or bool(self.queue) - - def next_request(self) -> Request | None: - if self.paused: - return None - try: - return self.queue.pop() - except IndexError: - return None - - def pause(self) -> None: - self.paused = True - - def unpause(self) -> None: - self.paused = False - - class MockSlot(NamedTuple): active: list[Any] diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py index 73c0e3776..b0632a7ea 100644 --- a/tests/utils/__init__.py +++ b/tests/utils/__init__.py @@ -1,13 +1,19 @@ +from __future__ import annotations + import asyncio import os -from collections.abc import Callable from pathlib import Path +from typing import TYPE_CHECKING from twisted.internet.defer import Deferred +from scrapy.settings import Settings, default_settings from scrapy.utils.asyncio import is_asyncio_available from scrapy.utils.defer import maybe_deferred_to_future +if TYPE_CHECKING: + from collections.abc import Callable + def twisted_sleep(seconds: float): from twisted.internet import reactor @@ -48,3 +54,8 @@ class OneShotLoop: def stop(self) -> None: self.running = False + + +def assert_option_is_default(settings: Settings, key: str) -> None: + assert isinstance(settings, Settings) + assert settings[key] == getattr(default_settings, key) diff --git a/tests/utils/base_commands.py b/tests/utils/base_commands.py new file mode 100644 index 000000000..594544c83 --- /dev/null +++ b/tests/utils/base_commands.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from shutil import copytree +from typing import TYPE_CHECKING + +import pytest + +from tests.utils.cmdline import call + +if TYPE_CHECKING: + from pathlib import Path + + +class TestProjectBase: + """A base class for tests that may need a Scrapy project.""" + + project_name = "testproject" + + @pytest.fixture(scope="session") + def _proj_path_cached(self, tmp_path_factory: pytest.TempPathFactory) -> Path: + """Create a Scrapy project in a temporary directory and return its path. + + Used as a cache for ``proj_path``. + """ + tmp_path = tmp_path_factory.mktemp("proj") + call("startproject", self.project_name, cwd=tmp_path) + return tmp_path / self.project_name + + @pytest.fixture + def proj_path(self, tmp_path: Path, _proj_path_cached: Path) -> Path: + """Copy a pre-generated Scrapy project into a temporary directory and return its path.""" + proj_path = tmp_path / self.project_name + copytree(_proj_path_cached, proj_path) + return proj_path diff --git a/tests/utils/cmdline.py b/tests/utils/cmdline.py index 122e0236d..62dff3d4c 100644 --- a/tests/utils/cmdline.py +++ b/tests/utils/cmdline.py @@ -2,12 +2,15 @@ from __future__ import annotations import subprocess import sys -from typing import Any +from typing import TYPE_CHECKING, Any import pytest from scrapy.utils.test import get_testenv +if TYPE_CHECKING: + from pathlib import Path + def call(*args: str, **popen_kwargs: Any) -> int: args = (sys.executable, "-m", "scrapy.cmdline", *args) @@ -36,3 +39,10 @@ def proc(*args: str, **popen_kwargs: Any) -> tuple[int, str, str]: pytest.fail("Command took too much time to complete") return p.returncode, p.stdout, p.stderr + + +def write_recording_editor(editor: Path) -> None: + """Create an executable editor script that writes the path it is asked to + open (its last argument) into the file given as its first argument.""" + editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8") + editor.chmod(0o755) diff --git a/tests/utils/feedexport.py b/tests/utils/feedexport.py new file mode 100644 index 000000000..515082217 --- /dev/null +++ b/tests/utils/feedexport.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from urllib.parse import urljoin +from urllib.request import pathname2url + +from scrapy import Field, Item + +if TYPE_CHECKING: + from pathlib import Path + + +def path_to_url(path: str | Path) -> str: + return urljoin("file:", pathname2url(str(path))) + + +def printf_escape(s: str) -> str: + return s.replace("%", "%%") + + +class MyItem(Item): + foo = Field() + egg = Field() + baz = Field() + + +class MyItem2(Item): + foo = Field() + hello = Field() diff --git a/tests/utils/redirect.py b/tests/utils/redirect.py new file mode 100644 index 000000000..a3a572988 --- /dev/null +++ b/tests/utils/redirect.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from itertools import product + +SCHEME_PARAMS = ("url", "location", "target") +HTTP_SCHEMES = ("http", "https") +NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo") +REDIRECT_SCHEME_CASES = ( + # http/https → http/https redirects + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + f"{output_scheme}://example.com/b", + ) + for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2) + ), + # http/https → data/file/ftp/s3/foo does not redirect + *( + ( + f"{input_scheme}://example.com/a", + f"{output_scheme}://example.com/b", + None, + ) + for input_scheme in HTTP_SCHEMES + for output_scheme in NON_HTTP_SCHEMES + ), + # http/https → relative redirects + *( + ( + f"{scheme}://example.com/a", + location, + f"{scheme}://example.com/b", + ) + for scheme in HTTP_SCHEMES + for location in ("//example.com/b", "/b") + ), + # Note: We do not test data/file/ftp/s3 schemes for the initial URL + # because their download handlers cannot return a status code of 3xx. +) diff --git a/tests/utils/robotstxt.py b/tests/utils/robotstxt.py new file mode 100644 index 000000000..c1b2f28c6 --- /dev/null +++ b/tests/utils/robotstxt.py @@ -0,0 +1,12 @@ +from __future__ import annotations + + +def rerp_available() -> bool: + # check if robotexclusionrulesparser is installed + try: + from robotexclusionrulesparser import ( # noqa: PLC0415 + RobotExclusionRulesParser, # noqa: F401 + ) + except ImportError: + return False + return True