RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge 7e1b2e6a78 into d8ba1571e7

This commit is contained in:
Andrey Rakhmatullin 2026-07-14 20:00:57 +00:00 committed by GitHub
parent d8ba1571e7 7e1b2e6a78
commit 31f64bee87
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
29 changed files with 796 additions and 712 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
tests/spiders.py
View File

@ -571,3 +571,19 @@ class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider):
def headers_received(self, headers, body_length, request, spider):
self.meta["headers_received"] = headers
raise StopDownload(fail=True)
class ExceptionSpider(Spider):
name = "exception"
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
raise ValueError("Exception in from_crawler method")
class NoRequestsSpider(Spider):
name = "no_request"
async def start(self):
return
yield

2
tests/test_command_check.py
View File

@ -7,7 +7,7 @@ from unittest import TestCase
from unittest.mock import MagicMock, Mock, PropertyMock, call, patch
from scrapy.commands.check import Command, TextTestResult
from tests.test_commands import TestProjectBase
from tests.utils.base_commands import TestProjectBase
from tests.utils.cmdline import proc
if TYPE_CHECKING:

2
tests/test_command_crawl.py
View File

@ -2,7 +2,7 @@ from __future__ import annotations
from typing import TYPE_CHECKING
from tests.test_commands import TestProjectBase
from tests.utils.base_commands import TestProjectBase
from tests.utils.cmdline import proc
if TYPE_CHECKING:

11
tests/test_command_genspider.py
View File

@ -6,15 +6,8 @@ from pathlib import Path
import pytest
from tests.test_commands import TestProjectBase
from tests.utils.cmdline import call, proc
def write_recording_editor(editor: Path) -> None:
"""Create an executable editor script that writes the path it is asked to
open (its last argument) into the file given as its first argument."""
editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
editor.chmod(0o755)
from tests.utils.base_commands import TestProjectBase
from tests.utils.cmdline import call, proc, write_recording_editor
def find_in_file(filename: Path, regex: str) -> re.Match[str] | None:

2
tests/test_command_parse.py
View File

@ -8,7 +8,7 @@ import pytest
from scrapy.commands import parse
from scrapy.settings import Settings
from tests.test_commands import TestProjectBase
from tests.utils.base_commands import TestProjectBase
from tests.utils.cmdline import call, proc
if TYPE_CHECKING:

7
tests/test_command_runspider.py
View File

@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
import pytest
from tests.test_crawler import ExceptionSpider, NoRequestsSpider
from tests.spiders import ExceptionSpider, NoRequestsSpider
from tests.utils.cmdline import proc
if TYPE_CHECKING:
@ -65,13 +65,14 @@ class BadSpider(scrapy.Spider):
def test_run_fail_spider(self, tmp_path: Path) -> None:
ret, _, _ = self.runspider(
tmp_path, "import scrapy\n" + inspect.getsource(ExceptionSpider)
tmp_path, "from scrapy import Spider\n" + inspect.getsource(ExceptionSpider)
)
assert ret != 0
def test_run_good_spider(self, tmp_path: Path) -> None:
ret, _, _ = self.runspider(
tmp_path, "import scrapy\n" + inspect.getsource(NoRequestsSpider)
tmp_path,
"from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider),
)
assert ret == 0

31
tests/test_commands.py
View File

@ -4,7 +4,6 @@ import argparse
import json
import sys
from io import StringIO
from shutil import copytree
from typing import TYPE_CHECKING
from unittest import mock
@ -16,7 +15,8 @@ from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
from scrapy.utils.reactor import _asyncio_reactor_path
from tests.utils.cmdline import call, proc
from tests.utils.base_commands import TestProjectBase
from tests.utils.cmdline import call, proc, write_recording_editor
if TYPE_CHECKING:
from pathlib import Path
@ -109,29 +109,6 @@ class TestCommandSettings:
)
class TestProjectBase:
"""A base class for tests that may need a Scrapy project."""
project_name = "testproject"
@pytest.fixture(scope="session")
def _proj_path_cached(self, tmp_path_factory: pytest.TempPathFactory) -> Path:
"""Create a Scrapy project in a temporary directory and return its path.
Used as a cache for ``proj_path``.
"""
tmp_path = tmp_path_factory.mktemp("proj")
call("startproject", self.project_name, cwd=tmp_path)
return tmp_path / self.project_name
@pytest.fixture
def proj_path(self, tmp_path: Path, _proj_path_cached: Path) -> Path:
"""Copy a pre-generated Scrapy project into a temporary directory and return its path."""
proj_path = tmp_path / self.project_name
copytree(_proj_path_cached, proj_path)
return proj_path
class TestCommandCrawlerProcess(TestProjectBase):
"""Test that the command uses the expected kind of *CrawlerProcess
and produces expected errors when needed."""
@ -430,9 +407,7 @@ class TestEditCommand(TestProjectBase):
spider = proj_path / self.project_name / "spiders" / "example.py"
edited = proj_path / "edited.txt"
editor = proj_path / "fake-editor.sh"
# Records the file it is asked to open ($2) into the file given as $1.
editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
editor.chmod(0o755)
write_recording_editor(editor)
monkeypatch.setenv("EDITOR", f"{editor} {edited}")
assert call("genspider", "example", "example.com", cwd=proj_path) == 0

49
tests/test_crawl.py
View File

@ -14,14 +14,13 @@ from twisted.internet.ssl import Certificate
from twisted.python.failure import Failure
from scrapy import Spider, signals
from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner
from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload
from scrapy.http import Request
from scrapy.http.response import Response
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.engine import format_engine_status, get_engine_status
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler, get_reactor_settings
from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
from tests.spiders import (
AsyncDefAsyncioGenComplexSpider,
@ -430,50 +429,6 @@ with multiples lines
)
assert not crawler.crawling
@coroutine_test
async def test_crawlerrunner_accepts_crawler(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
) -> None:
crawler = Crawler(SimpleSpider, get_reactor_settings())
runner = CrawlerRunner()
with caplog.at_level(logging.DEBUG):
await maybe_deferred_to_future(
runner.crawl(
crawler,
mockserver.url("/status?n=200"),
mockserver=mockserver,
)
)
assert "Got response 200" in caplog.text
@coroutine_test
async def test_crawl_multiple(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
) -> None:
settings_dict = get_reactor_settings()
runner_cls = (
CrawlerRunner
if settings_dict.get("TWISTED_REACTOR_ENABLED", True)
else AsyncCrawlerRunner
)
runner = runner_cls(settings_dict)
runner.crawl(
SimpleSpider,
mockserver.url("/status?n=200"),
mockserver=mockserver,
)
runner.crawl(
SimpleSpider,
mockserver.url("/status?n=503"),
mockserver=mockserver,
)
with caplog.at_level(logging.DEBUG):
await ensure_awaitable(runner.join())
self._assert_retried(caplog.text)
assert "Got response 200" in caplog.text
@coroutine_test
async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None:
crawler = get_crawler(SimpleSpider)

254
tests/test_crawler.py
View File

@ -10,22 +10,14 @@ from typing import TYPE_CHECKING, Any, ClassVar
from unittest.mock import MagicMock
import pytest
from zope.interface.exceptions import MultipleInvalid
import scrapy
from scrapy import Spider
from scrapy.crawler import (
AsyncCrawlerProcess,
AsyncCrawlerRunner,
Crawler,
CrawlerProcess,
CrawlerRunner,
CrawlerRunnerBase,
)
from scrapy.crawler import AsyncCrawlerProcess, Crawler, CrawlerProcess
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.throttle import AutoThrottle
from scrapy.settings import Settings, default_settings
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.log import (
_uninstall_scrapy_root_handler,
configure_logging,
@ -33,6 +25,8 @@ from scrapy.utils.log import (
)
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests.spiders import NoRequestsSpider
from tests.utils import assert_option_is_default
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
@ -52,14 +46,7 @@ def get_raw_crawler(
return Crawler(spidercls or DefaultSpider, settings)
class TestBaseCrawler:
@staticmethod
def assertOptionIsDefault(settings: Settings, key: str) -> None:
assert isinstance(settings, Settings)
assert settings[key] == getattr(default_settings, key)
class TestCrawler(TestBaseCrawler):
class TestCrawler:
def test_populate_spidercls_settings(self) -> None:
spider_settings: dict[str, Any] = {
"TEST1": "spider",
@ -90,11 +77,11 @@ class TestCrawler(TestBaseCrawler):
def test_crawler_accepts_dict(self) -> None:
crawler = get_crawler(DefaultSpider, {"foo": "bar"})
assert crawler.settings["foo"] == "bar"
self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
assert_option_is_default(crawler.settings, "RETRY_ENABLED")
def test_crawler_accepts_None(self) -> None:
crawler = Crawler(DefaultSpider)
self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
assert_option_is_default(crawler.settings, "RETRY_ENABLED")
def test_crawler_rejects_spider_objects(self) -> None:
with pytest.raises(ValueError, match="spidercls argument must be a class"):
@ -586,78 +573,6 @@ class TestCrawlerLogging:
assert "debug message" in logged
class SpiderLoaderWithWrongInterface:
def unneeded_method(self) -> None:
pass
class TestCrawlerRunner(TestBaseCrawler):
def test_spider_manager_verify_interface(self) -> None:
settings = Settings(
{
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
}
)
with pytest.raises(MultipleInvalid):
CrawlerRunner(settings)
def test_crawler_runner_accepts_dict(self) -> None:
runner = CrawlerRunner({"foo": "bar"})
assert runner.settings["foo"] == "bar"
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
def test_crawler_runner_accepts_None(self) -> None:
runner = CrawlerRunner()
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
class TestAsyncCrawlerRunner(TestBaseCrawler):
def test_spider_manager_verify_interface(self) -> None:
settings = Settings(
{
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
}
)
with pytest.raises(MultipleInvalid):
AsyncCrawlerRunner(settings)
def test_crawler_runner_accepts_dict(self) -> None:
runner = AsyncCrawlerRunner({"foo": "bar"})
assert runner.settings["foo"] == "bar"
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
def test_crawler_runner_accepts_None(self) -> None:
runner = AsyncCrawlerRunner()
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
class TestCrawlerProcess(TestBaseCrawler):
def test_crawler_process_accepts_dict(self) -> None:
runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)
assert runner.settings["foo"] == "bar"
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
def test_crawler_process_accepts_None(self) -> None:
runner = CrawlerProcess(install_root_handler=False)
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
@pytest.mark.only_asyncio
class TestAsyncCrawlerProcess(TestBaseCrawler):
def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:
runner = AsyncCrawlerProcess(
{"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},
install_root_handler=False,
)
assert runner.settings["foo"] == "bar"
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
@pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False
def test_crawler_process_accepts_None(self) -> None:
runner = AsyncCrawlerProcess(install_root_handler=False)
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
class TestAsyncCrawlerProcessReactorlessHelpers:
"""Unit tests for the reactorless shutdown helpers of AsyncCrawlerProcess.
@ -797,161 +712,6 @@ class TestAsyncCrawlerProcessReactorlessHelpers:
)
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
def test_runner_settings_applied_to_crawler_instance(
runner_cls: type[CrawlerRunnerBase],
) -> None:
runner = runner_cls({"FOO": "runner"})
crawler = Crawler(DefaultSpider)
result = runner.create_crawler(crawler)
assert result is crawler
assert result.settings["FOO"] == "runner"
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
def test_spider_custom_settings_override_runner(
runner_cls: type[CrawlerRunnerBase],
) -> None:
class MySpider(DefaultSpider):
custom_settings = {"FOO": "spider"}
runner = runner_cls({"FOO": "runner"})
crawler = Crawler(MySpider)
runner.create_crawler(crawler)
assert crawler.settings["FOO"] == "spider"
def test_create_crawler_instance_consistent_with_spider_class() -> None:
runner = AsyncCrawlerRunner({"FOO": "runner"})
crawler_from_class = runner.create_crawler(DefaultSpider)
pre_built = Crawler(DefaultSpider)
runner.create_crawler(pre_built)
assert crawler_from_class.settings["FOO"] == "runner"
assert pre_built.settings["FOO"] == "runner"
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
def test_create_crawler_rejects_spider_object(
runner_cls: type[CrawlerRunnerBase],
) -> None:
runner = runner_cls()
with pytest.raises(ValueError, match="cannot be a spider object"):
runner.create_crawler(DefaultSpider()) # type: ignore[arg-type]
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None:
runner = runner_cls()
with pytest.raises(ValueError, match="cannot be a spider object"):
runner.crawl(DefaultSpider()) # type: ignore[arg-type]
class ExceptionSpider(scrapy.Spider):
name = "exception"
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
raise ValueError("Exception in from_crawler method")
class NoRequestsSpider(scrapy.Spider):
name = "no_request"
async def start(self):
return
yield
@pytest.mark.requires_reactor # CrawlerRunner requires a reactor
class TestCrawlerRunnerHasSpider:
@pytest.fixture
def runner(self) -> CrawlerRunnerBase:
return CrawlerRunner(get_reactor_settings())
@staticmethod
async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:
await ensure_awaitable(runner.crawl(spider))
@coroutine_test
async def test_crawler_runner_bootstrap_successful(
self, runner: CrawlerRunnerBase
) -> None:
await self._crawl(runner, NoRequestsSpider)
assert not runner.bootstrap_failed
@coroutine_test
async def test_crawler_runner_bootstrap_successful_for_several(
self, runner: CrawlerRunnerBase
) -> None:
await self._crawl(runner, NoRequestsSpider)
await self._crawl(runner, NoRequestsSpider)
assert not runner.bootstrap_failed
@coroutine_test
async def test_crawler_runner_bootstrap_failed(
self, runner: CrawlerRunnerBase
) -> None:
try:
await self._crawl(runner, ExceptionSpider)
except ValueError:
pass
else:
pytest.fail("Exception should be raised from spider")
assert runner.bootstrap_failed
@coroutine_test
async def test_crawler_runner_bootstrap_failed_for_several(
self, runner: CrawlerRunnerBase
) -> None:
try:
await self._crawl(runner, ExceptionSpider)
except ValueError:
pass
else:
pytest.fail("Exception should be raised from spider")
await self._crawl(runner, NoRequestsSpider)
assert runner.bootstrap_failed
@coroutine_test
async def test_crawler_runner_asyncio_enabled_true(
self, reactor_pytest: str
) -> None:
if reactor_pytest != "asyncio":
runner = CrawlerRunner(
settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
)
with pytest.raises(
Exception,
match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
):
await self._crawl(runner, NoRequestsSpider)
else:
runner = CrawlerRunner(
settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
)
await self._crawl(runner, NoRequestsSpider)
@pytest.mark.only_asyncio
class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
@pytest.fixture
def runner(self) -> CrawlerRunnerBase:
return AsyncCrawlerRunner(get_reactor_settings())
def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override]
pytest.skip("This test is only for CrawlerRunner")
@pytest.mark.parametrize(
("settings", "items"),
[

284
tests/test_crawler_runners.py Normal file
View File

@ -0,0 +1,284 @@
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
import pytest
from zope.interface.exceptions import MultipleInvalid
from scrapy.crawler import (
AsyncCrawlerProcess,
AsyncCrawlerRunner,
Crawler,
CrawlerProcess,
CrawlerRunner,
CrawlerRunnerBase,
)
from scrapy.settings import Settings
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_reactor_settings
from tests.spiders import ExceptionSpider, NoRequestsSpider, SimpleSpider
from tests.utils import assert_option_is_default
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy import Spider
from tests.mockserver.http import MockServer
class SpiderLoaderWithWrongInterface:
def unneeded_method(self) -> None:
pass
class TestCrawlerRunner:
def test_spider_manager_verify_interface(self) -> None:
settings = Settings(
{
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
}
)
with pytest.raises(MultipleInvalid):
CrawlerRunner(settings)
def test_crawler_runner_accepts_dict(self) -> None:
runner = CrawlerRunner({"foo": "bar"})
assert runner.settings["foo"] == "bar"
assert_option_is_default(runner.settings, "RETRY_ENABLED")
def test_crawler_runner_accepts_None(self) -> None:
runner = CrawlerRunner()
assert_option_is_default(runner.settings, "RETRY_ENABLED")
class TestAsyncCrawlerRunner:
def test_spider_manager_verify_interface(self) -> None:
settings = Settings(
{
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
}
)
with pytest.raises(MultipleInvalid):
AsyncCrawlerRunner(settings)
def test_crawler_runner_accepts_dict(self) -> None:
runner = AsyncCrawlerRunner({"foo": "bar"})
assert runner.settings["foo"] == "bar"
assert_option_is_default(runner.settings, "RETRY_ENABLED")
def test_crawler_runner_accepts_None(self) -> None:
runner = AsyncCrawlerRunner()
assert_option_is_default(runner.settings, "RETRY_ENABLED")
class TestCrawlerProcess:
def test_crawler_process_accepts_dict(self) -> None:
runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)
assert runner.settings["foo"] == "bar"
assert_option_is_default(runner.settings, "RETRY_ENABLED")
def test_crawler_process_accepts_None(self) -> None:
runner = CrawlerProcess(install_root_handler=False)
assert_option_is_default(runner.settings, "RETRY_ENABLED")
@pytest.mark.only_asyncio
class TestAsyncCrawlerProcess:
def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:
runner = AsyncCrawlerProcess(
{"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},
install_root_handler=False,
)
assert runner.settings["foo"] == "bar"
assert_option_is_default(runner.settings, "RETRY_ENABLED")
@pytest.mark.requires_reactor # can't pass TWISTED_REACTOR_ENABLED=False
def test_crawler_process_accepts_None(self) -> None:
runner = AsyncCrawlerProcess(install_root_handler=False)
assert_option_is_default(runner.settings, "RETRY_ENABLED")
@pytest.mark.requires_reactor # CrawlerRunner requires a reactor
class TestCrawlerRunnerHasSpider:
@pytest.fixture
def runner(self) -> CrawlerRunnerBase:
return CrawlerRunner(get_reactor_settings())
@staticmethod
async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:
await ensure_awaitable(runner.crawl(spider))
@coroutine_test
async def test_crawler_runner_bootstrap_successful(
self, runner: CrawlerRunnerBase
) -> None:
await self._crawl(runner, NoRequestsSpider)
assert not runner.bootstrap_failed
@coroutine_test
async def test_crawler_runner_bootstrap_successful_for_several(
self, runner: CrawlerRunnerBase
) -> None:
await self._crawl(runner, NoRequestsSpider)
await self._crawl(runner, NoRequestsSpider)
assert not runner.bootstrap_failed
@coroutine_test
async def test_crawler_runner_bootstrap_failed(
self, runner: CrawlerRunnerBase
) -> None:
try:
await self._crawl(runner, ExceptionSpider)
except ValueError:
pass
else:
pytest.fail("Exception should be raised from spider")
assert runner.bootstrap_failed
@coroutine_test
async def test_crawler_runner_bootstrap_failed_for_several(
self, runner: CrawlerRunnerBase
) -> None:
try:
await self._crawl(runner, ExceptionSpider)
except ValueError:
pass
else:
pytest.fail("Exception should be raised from spider")
await self._crawl(runner, NoRequestsSpider)
assert runner.bootstrap_failed
@coroutine_test
async def test_crawler_runner_asyncio_enabled_true(
self, reactor_pytest: str
) -> None:
if reactor_pytest != "asyncio":
runner = CrawlerRunner(
settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
)
with pytest.raises(
Exception,
match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
):
await self._crawl(runner, NoRequestsSpider)
else:
runner = CrawlerRunner(
settings={
"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
}
)
await self._crawl(runner, NoRequestsSpider)
@pytest.mark.only_asyncio
class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
@pytest.fixture
def runner(self) -> CrawlerRunnerBase:
return AsyncCrawlerRunner(get_reactor_settings())
def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override]
pytest.skip("This test is only for CrawlerRunner")
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
def test_runner_settings_applied_to_crawler_instance(
runner_cls: type[CrawlerRunnerBase],
) -> None:
runner = runner_cls({"FOO": "runner"})
crawler = Crawler(DefaultSpider)
result = runner.create_crawler(crawler)
assert result is crawler
assert result.settings["FOO"] == "runner"
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
def test_spider_custom_settings_override_runner(
runner_cls: type[CrawlerRunnerBase],
) -> None:
class MySpider(DefaultSpider):
custom_settings = {"FOO": "spider"}
runner = runner_cls({"FOO": "runner"})
crawler = Crawler(MySpider)
runner.create_crawler(crawler)
assert crawler.settings["FOO"] == "spider"
def test_create_crawler_instance_consistent_with_spider_class() -> None:
runner = AsyncCrawlerRunner({"FOO": "runner"})
crawler_from_class = runner.create_crawler(DefaultSpider)
pre_built = Crawler(DefaultSpider)
runner.create_crawler(pre_built)
assert crawler_from_class.settings["FOO"] == "runner"
assert pre_built.settings["FOO"] == "runner"
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
def test_create_crawler_rejects_spider_object(
runner_cls: type[CrawlerRunnerBase],
) -> None:
runner = runner_cls()
with pytest.raises(ValueError, match="cannot be a spider object"):
runner.create_crawler(DefaultSpider()) # type: ignore[arg-type]
@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None:
runner = runner_cls()
with pytest.raises(ValueError, match="cannot be a spider object"):
runner.crawl(DefaultSpider()) # type: ignore[arg-type]
@coroutine_test
async def test_crawlerrunner_accepts_crawler(
caplog: pytest.LogCaptureFixture, mockserver: MockServer
) -> None:
crawler = Crawler(SimpleSpider, get_reactor_settings())
runner = CrawlerRunner()
with caplog.at_level(logging.DEBUG):
await maybe_deferred_to_future(
runner.crawl(
crawler,
mockserver.url("/status?n=200"),
mockserver=mockserver,
)
)
assert "Got response 200" in caplog.text
@coroutine_test
async def test_crawl_multiple(
caplog: pytest.LogCaptureFixture, mockserver: MockServer
) -> None:
settings_dict = get_reactor_settings()
runner_cls = (
CrawlerRunner
if settings_dict.get("TWISTED_REACTOR_ENABLED", True)
else AsyncCrawlerRunner
)
runner = runner_cls(settings_dict)
runner.crawl(
SimpleSpider,
mockserver.url("/status?n=200"),
mockserver=mockserver,
)
runner.crawl(
SimpleSpider,
mockserver.url("/status?n=503"),
mockserver=mockserver,
)
with caplog.at_level(logging.DEBUG):
await ensure_awaitable(runner.join())
assert "Got response 200" in caplog.text
assert "Gave up retrying" in caplog.text

7
tests/test_downloadermiddleware_redirect.py
View File

@ -16,11 +16,8 @@ from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.test_downloadermiddleware_redirect_base import (
REDIRECT_SCHEME_CASES,
SCHEME_PARAMS,
Base,
)
from tests.test_downloadermiddleware_redirect_base import Base
from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMS
class TestRedirectMiddleware(Base.Test):

39
tests/test_downloadermiddleware_redirect_base.py
View File

@ -1,7 +1,5 @@
from __future__ import annotations
from itertools import product
import pytest
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
@ -10,43 +8,6 @@ from scrapy.http import Request, Response
from scrapy.utils.misc import set_environ
from scrapy.utils.test import get_crawler
SCHEME_PARAMS = ("url", "location", "target")
HTTP_SCHEMES = ("http", "https")
NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo")
REDIRECT_SCHEME_CASES = (
# http/https → http/https redirects
*(
(
f"{input_scheme}://example.com/a",
f"{output_scheme}://example.com/b",
f"{output_scheme}://example.com/b",
)
for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2)
),
# http/https → data/file/ftp/s3/foo does not redirect
*(
(
f"{input_scheme}://example.com/a",
f"{output_scheme}://example.com/b",
None,
)
for input_scheme in HTTP_SCHEMES
for output_scheme in NON_HTTP_SCHEMES
),
# http/https → relative redirects
*(
(
f"{scheme}://example.com/a",
location,
f"{scheme}://example.com/b",
)
for scheme in HTTP_SCHEMES
for location in ("//example.com/b", "/b")
),
# Note: We do not test data/file/ftp/s3 schemes for the initial URL
# because their download handlers cannot return a status code of 3xx.
)
class Base:
class Test:

4
tests/test_downloadermiddleware_redirect_metarefresh.py
View File

@ -12,12 +12,12 @@ from scrapy.http import HtmlResponse, Request, Response
from scrapy.spiders import Spider
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.test_downloadermiddleware_redirect_base import (
from tests.test_downloadermiddleware_redirect_base import Base
from tests.utils.redirect import (
HTTP_SCHEMES,
NON_HTTP_SCHEMES,
REDIRECT_SCHEME_CASES,
SCHEME_PARAMS,
Base,
)

2
tests/test_downloadermiddleware_robotstxt.py
View File

@ -15,8 +15,8 @@ from scrapy.http.request import NO_CALLBACK
from scrapy.settings import Settings
from scrapy.utils.asyncio import call_later
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from tests.test_robotstxt_interface import rerp_available
from tests.utils.decorators import coroutine_test
from tests.utils.robotstxt import rerp_available
if TYPE_CHECKING:
from scrapy.crawler import Crawler

228
tests/test_engine.py
View File

@ -6,8 +6,8 @@ import subprocess
import sys
from collections import defaultdict
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
from unittest.mock import Mock, call
from typing import TYPE_CHECKING, Any
from unittest.mock import Mock
from urllib.parse import urlparse
import attr
@ -25,7 +25,6 @@ from scrapy.http import Headers, Request, Response
from scrapy.item import Field, Item
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Spider
from scrapy.statscollectors import MemoryStatsCollector
from scrapy.utils.defer import (
_schedule_coro,
deferred_from_coro,
@ -42,8 +41,6 @@ if TYPE_CHECKING:
from twisted.python.failure import Failure
from scrapy.core.scheduler import Scheduler
from scrapy.crawler import Crawler
from tests.mockserver.http import MockServer
@ -515,98 +512,6 @@ class TestEngine(TestEngineBase):
assert "AssertionError" not in stderr_str, stderr_str
class TestEngineDownloadAsync:
"""Test cases for ExecutionEngine.download_async()."""
@pytest.fixture
def engine(self) -> ExecutionEngine:
crawler = get_crawler(MySpider)
engine = ExecutionEngine(crawler, lambda _: None)
engine.downloader.close()
engine.downloader = Mock()
engine._slot = Mock()
engine._slot.inprogress = set()
return engine
@staticmethod
async def _download(engine: ExecutionEngine, request: Request) -> Response:
return await engine.download_async(request)
@coroutine_test
async def test_download_async_success(self, engine):
"""Test basic successful async download of a request."""
request = Request("http://example.com")
response = Response("http://example.com", body=b"test body")
engine.spider = Mock()
engine.downloader.fetch.return_value = defer.succeed(response)
engine._slot.add_request = Mock()
engine._slot.remove_request = Mock()
result = await self._download(engine, request)
assert result == response
engine._slot.add_request.assert_called_once_with(request)
engine._slot.remove_request.assert_called_once_with(request)
engine.downloader.fetch.assert_called_once_with(request)
@coroutine_test
async def test_download_async_redirect(self, engine):
"""Test async download with a redirect request."""
original_request = Request("http://example.com")
redirect_request = Request("http://example.com/redirect")
final_response = Response("http://example.com/redirect", body=b"redirected")
# First call returns redirect request, second call returns final response
engine.downloader.fetch.side_effect = [
defer.succeed(redirect_request),
defer.succeed(final_response),
]
engine.spider = Mock()
engine._slot.add_request = Mock()
engine._slot.remove_request = Mock()
result = await self._download(engine, original_request)
assert result == final_response
assert engine.downloader.fetch.call_count == 2
engine._slot.add_request.assert_has_calls(
[call(original_request), call(redirect_request)]
)
engine._slot.remove_request.assert_has_calls(
[call(original_request), call(redirect_request)]
)
@coroutine_test
async def test_download_async_no_spider(self, engine):
"""Test async download attempt when no spider is available."""
request = Request("http://example.com")
engine.spider = None
with pytest.raises(RuntimeError, match="No open spider to crawl:"):
await self._download(engine, request)
@coroutine_test
async def test_download_async_failure(self, engine):
"""Test async download when the downloader raises an exception."""
request = Request("http://example.com")
error = RuntimeError("Download failed")
engine.spider = Mock()
engine.downloader.fetch.return_value = defer.fail(error)
engine._slot.add_request = Mock()
engine._slot.remove_request = Mock()
with pytest.raises(RuntimeError, match="Download failed"):
await self._download(engine, request)
engine._slot.add_request.assert_called_once_with(request)
engine._slot.remove_request.assert_called_once_with(request)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestEngineDownload(TestEngineDownloadAsync):
"""Test cases for ExecutionEngine.download()."""
@staticmethod
async def _download(engine: ExecutionEngine, request: Request) -> Response:
return await maybe_deferred_to_future(engine.download(request))
@coroutine_test
async def test_request_scheduled_signal():
class TestScheduler(BaseScheduler):
@ -640,132 +545,3 @@ async def test_request_scheduled_signal():
f"{scheduler.enqueued!r} != [{keep_request!r}]"
)
crawler.signals.disconnect(signal_handler, signals.request_scheduled)
class TestEngineCloseSpider:
"""Tests for exception handling coverage during close_spider_async()."""
@pytest.fixture
def crawler(self) -> Crawler:
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
return crawler
@coroutine_test
async def test_no_slot(self, crawler: Crawler) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
slot = engine._slot
engine._slot = None
with pytest.raises(RuntimeError, match="Engine slot not assigned"):
await engine.close_spider_async()
# close it correctly
engine._slot = slot
await engine.close_spider_async()
@coroutine_test
async def test_no_spider(self, crawler: Crawler) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
with pytest.raises(RuntimeError, match="Spider not opened"):
await engine.close_spider_async()
engine.downloader.close() # cleanup
@coroutine_test
async def test_exception_slot(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
assert engine._slot
del engine._slot.heartbeat
await engine.close_spider_async()
assert "Slot close failure" in caplog.text
@coroutine_test
async def test_exception_downloader(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
del engine.downloader.slots
await engine.close_spider_async()
assert "Downloader close failure" in caplog.text
@coroutine_test
async def test_exception_scraper(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
engine.scraper.slot = None
await engine.close_spider_async()
assert "Scraper close failure" in caplog.text
@coroutine_test
async def test_exception_scheduler(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
assert engine._slot
del cast("Scheduler", engine._slot.scheduler).dqs
await engine.close_spider_async()
assert "Scheduler close failure" in caplog.text
@coroutine_test
async def test_exception_signal(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
signal_manager = engine.signals
del engine.signals
await engine.close_spider_async()
assert "Error while sending spider_close signal" in caplog.text
# send the spider_closed signal to close various components
await signal_manager.send_catch_log_async(
signal=signals.spider_closed,
spider=engine.spider,
reason="cancelled",
)
@coroutine_test
async def test_exception_stats(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
assert isinstance(crawler.stats, MemoryStatsCollector)
del crawler.stats.spider_stats
await engine.close_spider_async()
assert "Stats close failure" in caplog.text
@coroutine_test
async def test_exception_callback(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))
crawler.engine = engine
await engine.open_spider_async()
await engine.close_spider_async()
assert "Error running spider_closed_callback" in caplog.text
@coroutine_test
async def test_exception_async_callback(
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
async def cb(_):
raise ValueError
engine = ExecutionEngine(crawler, cb)
crawler.engine = engine
await engine.open_spider_async()
await engine.close_spider_async()
assert "Error running spider_closed_callback" in caplog.text

153
tests/test_engine_close_spider.py Normal file
View File

@ -0,0 +1,153 @@
from __future__ import annotations
from typing import TYPE_CHECKING, cast
import pytest
from twisted.internet import defer
from scrapy import signals
from scrapy.core.engine import ExecutionEngine
from scrapy.statscollectors import MemoryStatsCollector
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy.core.scheduler import Scheduler
from scrapy.crawler import Crawler
@pytest.fixture
def crawler() -> Crawler:
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
return crawler
@coroutine_test
async def test_no_slot(crawler: Crawler) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
slot = engine._slot
engine._slot = None
with pytest.raises(RuntimeError, match="Engine slot not assigned"):
await engine.close_spider_async()
# close it correctly
engine._slot = slot
await engine.close_spider_async()
@coroutine_test
async def test_no_spider(crawler: Crawler) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
with pytest.raises(RuntimeError, match="Spider not opened"):
await engine.close_spider_async()
engine.downloader.close() # cleanup
@coroutine_test
async def test_exception_slot(
crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
assert engine._slot
del engine._slot.heartbeat
await engine.close_spider_async()
assert "Slot close failure" in caplog.text
@coroutine_test
async def test_exception_downloader(
crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
del engine.downloader.slots
await engine.close_spider_async()
assert "Downloader close failure" in caplog.text
@coroutine_test
async def test_exception_scraper(
crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
engine.scraper.slot = None
await engine.close_spider_async()
assert "Scraper close failure" in caplog.text
@coroutine_test
async def test_exception_scheduler(
crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
assert engine._slot
del cast("Scheduler", engine._slot.scheduler).dqs
await engine.close_spider_async()
assert "Scheduler close failure" in caplog.text
@coroutine_test
async def test_exception_signal(
crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
signal_manager = engine.signals
del engine.signals
await engine.close_spider_async()
assert "Error while sending spider_close signal" in caplog.text
# send the spider_closed signal to close various components
await signal_manager.send_catch_log_async(
signal=signals.spider_closed,
spider=engine.spider,
reason="cancelled",
)
@coroutine_test
async def test_exception_stats(
crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
crawler.engine = engine
await engine.open_spider_async()
assert isinstance(crawler.stats, MemoryStatsCollector)
del crawler.stats.spider_stats
await engine.close_spider_async()
assert "Stats close failure" in caplog.text
@coroutine_test
async def test_exception_callback(
crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))
crawler.engine = engine
await engine.open_spider_async()
await engine.close_spider_async()
assert "Error running spider_closed_callback" in caplog.text
@coroutine_test
async def test_exception_async_callback(
crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
async def cb(_):
raise ValueError
engine = ExecutionEngine(crawler, cb)
crawler.engine = engine
await engine.open_spider_async()
await engine.close_spider_async()
assert "Error running spider_closed_callback" in caplog.text

106
tests/test_engine_download.py Normal file
View File

@ -0,0 +1,106 @@
from __future__ import annotations
from unittest.mock import Mock, call
import pytest
from twisted.internet import defer
from scrapy.core.engine import ExecutionEngine
from scrapy.http import Request, Response
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
@pytest.fixture
def engine() -> ExecutionEngine:
crawler = get_crawler(DefaultSpider)
engine = ExecutionEngine(crawler, lambda _: None)
engine.downloader.close()
engine.downloader = Mock()
engine._slot = Mock()
engine._slot.inprogress = set()
return engine
class TestEngineDownloadAsync:
"""Test cases for ExecutionEngine.download_async()."""
@staticmethod
async def _download(engine: ExecutionEngine, request: Request) -> Response:
return await engine.download_async(request)
@coroutine_test
async def test_download_async_success(self, engine):
"""Test basic successful async download of a request."""
request = Request("http://example.com")
response = Response("http://example.com", body=b"test body")
engine.spider = Mock()
engine.downloader.fetch.return_value = defer.succeed(response)
engine._slot.add_request = Mock()
engine._slot.remove_request = Mock()
result = await self._download(engine, request)
assert result == response
engine._slot.add_request.assert_called_once_with(request)
engine._slot.remove_request.assert_called_once_with(request)
engine.downloader.fetch.assert_called_once_with(request)
@coroutine_test
async def test_download_async_redirect(self, engine):
"""Test async download with a redirect request."""
original_request = Request("http://example.com")
redirect_request = Request("http://example.com/redirect")
final_response = Response("http://example.com/redirect", body=b"redirected")
# First call returns redirect request, second call returns final response
engine.downloader.fetch.side_effect = [
defer.succeed(redirect_request),
defer.succeed(final_response),
]
engine.spider = Mock()
engine._slot.add_request = Mock()
engine._slot.remove_request = Mock()
result = await self._download(engine, original_request)
assert result == final_response
assert engine.downloader.fetch.call_count == 2
engine._slot.add_request.assert_has_calls(
[call(original_request), call(redirect_request)]
)
engine._slot.remove_request.assert_has_calls(
[call(original_request), call(redirect_request)]
)
@coroutine_test
async def test_download_async_no_spider(self, engine):
"""Test async download attempt when no spider is available."""
request = Request("http://example.com")
engine.spider = None
with pytest.raises(RuntimeError, match="No open spider to crawl:"):
await self._download(engine, request)
@coroutine_test
async def test_download_async_failure(self, engine):
"""Test async download when the downloader raises an exception."""
request = Request("http://example.com")
error = RuntimeError("Download failed")
engine.spider = Mock()
engine.downloader.fetch.return_value = defer.fail(error)
engine._slot.add_request = Mock()
engine._slot.remove_request = Mock()
with pytest.raises(RuntimeError, match="Download failed"):
await self._download(engine, request)
engine._slot.add_request.assert_called_once_with(request)
engine._slot.remove_request.assert_called_once_with(request)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestEngineDownload(TestEngineDownloadAsync):
"""Test cases for ExecutionEngine.download()."""
@staticmethod
async def _download(engine: ExecutionEngine, request: Request) -> Response:
return await maybe_deferred_to_future(engine.download(request))

36
tests/test_engine_loop.py
View File

@ -2,13 +2,13 @@ from __future__ import annotations
from collections import deque
from logging import ERROR
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider, signals
from scrapy.core.scheduler import BaseScheduler
from scrapy.utils.asyncio import call_later
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.test_scheduler import MemoryScheduler
from tests.utils import async_sleep
from tests.utils.decorators import coroutine_test
@ -18,6 +18,38 @@ if TYPE_CHECKING:
from scrapy.http import Response
class MemoryScheduler(BaseScheduler):
paused = False
def __init__(self, *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
self.queue: deque[Request] = deque(
Request(value) if isinstance(value, str) else value
for value in getattr(self, "queue", [])
)
def enqueue_request(self, request: Request) -> bool:
self.queue.append(request)
return True
def has_pending_requests(self) -> bool:
return self.paused or bool(self.queue)
def next_request(self) -> Request | None:
if self.paused:
return None
try:
return self.queue.pop()
except IndexError:
return None
def pause(self) -> None:
self.paused = True
def unpause(self) -> None:
self.paused = False
class TestMain:
@coroutine_test
async def test_sleep(self):

64
tests/test_feedexport.py
View File

@ -13,8 +13,6 @@ from pathlib import Path
from string import ascii_letters, digits
from typing import IO, TYPE_CHECKING, Any
from unittest import mock
from urllib.parse import urljoin
from urllib.request import pathname2url
import lxml.etree
import pytest
@ -39,19 +37,12 @@ from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.spiders import ItemSpider
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from tests.utils.feedexport import MyItem, MyItem2, path_to_url, printf_escape
if TYPE_CHECKING:
from collections.abc import Awaitable, Callable, Iterable
def path_to_url(path: str | Path) -> str:
return urljoin("file:", pathname2url(str(path)))
def printf_escape(s: str) -> str:
return s.replace("%", "%%")
class FromCrawlerMixin:
init_with_crawler = False
@ -114,15 +105,6 @@ class LogOnStoreFileStorage:
class TestFeedExportBase(ABC):
mockserver: MockServer
class MyItem(scrapy.Item):
foo = scrapy.Field()
egg = scrapy.Field()
baz = scrapy.Field()
class MyItem2(scrapy.Item):
foo = scrapy.Field()
hello = scrapy.Field()
def _random_temp_filename(self, inter_dir="") -> Path:
chars = [random.choice(ascii_letters + digits) for _ in range(15)]
filename = "".join(chars)
@ -517,21 +499,21 @@ class TestFeedExport(TestFeedExportBase):
async def test_export_items(self):
# feed exporters use field names from Item
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
MyItem({"foo": "bar1", "egg": "spam1"}),
MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
]
rows = [
{"egg": "spam1", "foo": "bar1", "baz": ""},
{"egg": "spam2", "foo": "bar2", "baz": "quux2"},
]
header = self.MyItem.fields.keys()
header = MyItem.fields.keys()
await self.assertExported(items, header, rows)
@coroutine_test
async def test_pathlib_uri_with_placeholders(self):
feed_dir = Path(self.temp_dir, "pathlib_placeholders")
feed_dir.mkdir()
items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
items = [MyItem({"foo": "bar1", "egg": "spam1"})]
class TestSpider(scrapy.Spider):
name = "testspider"
@ -561,7 +543,7 @@ class TestFeedExport(TestFeedExportBase):
# so the resulting file name can be asserted exactly.
feed_dir = Path(self.temp_dir, "pathlib_spaces_unicode")
feed_dir.mkdir()
items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
items = [MyItem({"foo": "bar1", "egg": "spam1"})]
class TestSpider(scrapy.Spider):
name = "testspider"
@ -590,7 +572,7 @@ class TestFeedExport(TestFeedExportBase):
# and #5794.
feed_dir = Path(self.temp_dir, "dir with spaces")
feed_dir.mkdir()
items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
items = [MyItem({"foo": "bar1", "egg": "spam1"})]
class TestSpider(scrapy.Spider):
name = "testspider"
@ -627,7 +609,7 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_start_finish_exporting_items(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
MyItem({"foo": "bar1", "egg": "spam1"}),
]
settings = {
"FEEDS": {
@ -665,7 +647,7 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_start_finish_exporting_items_exception(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
MyItem({"foo": "bar1", "egg": "spam1"}),
]
settings = {
"FEEDS": {
@ -743,15 +725,15 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_export_multiple_item_classes(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
self.MyItem2({"hello": "world2", "foo": "bar2"}),
self.MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}),
MyItem({"foo": "bar1", "egg": "spam1"}),
MyItem2({"hello": "world2", "foo": "bar2"}),
MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}),
{"hello": "world4", "egg": "spam4"},
]
# by default, Scrapy uses fields of the first Item for CSV and
# all fields for JSON Lines
header = self.MyItem.fields.keys()
header = MyItem.fields.keys()
rows_csv = [
{"egg": "spam1", "foo": "bar1", "baz": ""},
{"egg": "", "foo": "bar2", "baz": ""},
@ -826,8 +808,8 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_export_based_on_item_classes(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
self.MyItem2({"hello": "world2", "foo": "bar2"}),
MyItem({"foo": "bar1", "egg": "spam1"}),
MyItem2({"hello": "world2", "foo": "bar2"}),
{"hello": "world3", "egg": "spam3"},
]
@ -849,15 +831,15 @@ class TestFeedExport(TestFeedExportBase):
"FEEDS": {
self._random_temp_filename(): {
"format": "csv",
"item_classes": [self.MyItem],
"item_classes": [MyItem],
},
self._random_temp_filename(): {
"format": "json",
"item_classes": [self.MyItem2],
"item_classes": [MyItem2],
},
self._random_temp_filename(): {
"format": "jsonlines",
"item_classes": [self.MyItem, self.MyItem2],
"item_classes": [MyItem, MyItem2],
},
self._random_temp_filename(): {
"format": "xml",
@ -872,13 +854,11 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_export_based_on_custom_filters(self):
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
self.MyItem2({"hello": "world2", "foo": "bar2"}),
MyItem({"foo": "bar1", "egg": "spam1"}),
MyItem2({"hello": "world2", "foo": "bar2"}),
{"hello": "world3", "egg": "spam3"},
]
MyItem = self.MyItem
class CustomFilter1:
def __init__(self, feed_options):
pass
@ -918,7 +898,7 @@ class TestFeedExport(TestFeedExportBase):
},
self._random_temp_filename(): {
"format": "jsonlines",
"item_classes": [self.MyItem, self.MyItem2],
"item_classes": [MyItem, MyItem2],
"item_filter": CustomFilter3,
},
},
@ -957,7 +937,7 @@ class TestFeedExport(TestFeedExportBase):
# FEED_EXPORT_FIELDS option allows to order export fields
# and to select a subset of fields to export, both for Items and dicts.
for item_cls in [self.MyItem, dict]:
for item_cls in [MyItem, dict]:
items = [
item_cls({"foo": "bar1", "egg": "spam1"}),
item_cls({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),

21
tests/test_feedexport_batch.py
View File

@ -24,6 +24,7 @@ from scrapy.utils.test import get_crawler
from tests.spiders import ItemSpider
from tests.test_feedexport import TestFeedExportBase
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from tests.utils.feedexport import MyItem
if TYPE_CHECKING:
from os import PathLike
@ -197,9 +198,9 @@ class TestBatchDeliveries(TestFeedExportBase):
async def test_export_items(self):
"""Test partial deliveries in all supported formats"""
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
self.MyItem({"foo": "bar3", "baz": "quux3"}),
MyItem({"foo": "bar1", "egg": "spam1"}),
MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
MyItem({"foo": "bar3", "baz": "quux3"}),
]
rows = [
{"egg": "spam1", "foo": "bar1", "baz": ""},
@ -207,7 +208,7 @@ class TestBatchDeliveries(TestFeedExportBase):
{"foo": "bar3", "baz": "quux3", "egg": ""},
]
settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2}
header = self.MyItem.fields.keys()
header = MyItem.fields.keys()
await self.assertExported(items, header, rows, settings=settings)
def test_wrong_path(self):
@ -349,9 +350,9 @@ class TestBatchDeliveries(TestFeedExportBase):
So %(batch_id)d replaced with the current id.
"""
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
self.MyItem({"foo": "bar3", "baz": "quux3"}),
MyItem({"foo": "bar1", "egg": "spam1"}),
MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
MyItem({"foo": "bar3", "baz": "quux3"}),
]
settings = {
"FEEDS": {
@ -387,9 +388,9 @@ class TestBatchDeliveries(TestFeedExportBase):
def test_s3_export(self):
bucket = "mybucket"
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
self.MyItem({"foo": "bar3", "baz": "quux3"}),
MyItem({"foo": "bar1", "egg": "spam1"}),
MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
MyItem({"foo": "bar3", "baz": "quux3"}),
]
class CustomS3FeedStorage(S3FeedStorage):

3
tests/test_feedexport_postprocess.py
View File

@ -13,8 +13,9 @@ from typing import TYPE_CHECKING, Any
import pytest
from scrapy.utils.test import get_crawler
from tests.test_feedexport import TestFeedExportBase, path_to_url, printf_escape
from tests.test_feedexport import TestFeedExportBase
from tests.utils.decorators import coroutine_test
from tests.utils.feedexport import path_to_url, printf_escape
if TYPE_CHECKING:
from scrapy import Spider

12
tests/test_robotstxt_interface.py
View File

@ -7,17 +7,7 @@ from scrapy.robotstxt import (
decode_robotstxt,
)
from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
def rerp_available() -> bool:
# check if robotexclusionrulesparser is installed
try:
from robotexclusionrulesparser import ( # noqa: PLC0415
RobotExclusionRulesParser, # noqa: F401
)
except ImportError:
return False
return True
from tests.utils.robotstxt import rerp_available
class BaseRobotParserTest:

35
tests/test_scheduler.py
View File

@ -2,7 +2,6 @@ from __future__ import annotations
import warnings
from abc import ABC, abstractmethod
from collections import deque
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from typing import TYPE_CHECKING, Any, NamedTuple, cast
from unittest.mock import Mock
@ -10,7 +9,7 @@ from unittest.mock import Mock
import pytest
from scrapy.core.downloader import Downloader
from scrapy.core.scheduler import BaseScheduler, Scheduler
from scrapy.core.scheduler import Scheduler
from scrapy.crawler import Crawler
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request
@ -27,38 +26,6 @@ if TYPE_CHECKING:
from pathlib import Path
class MemoryScheduler(BaseScheduler):
paused = False
def __init__(self, *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
self.queue: deque[Request] = deque(
Request(value) if isinstance(value, str) else value
for value in getattr(self, "queue", [])
)
def enqueue_request(self, request: Request) -> bool:
self.queue.append(request)
return True
def has_pending_requests(self) -> bool:
return self.paused or bool(self.queue)
def next_request(self) -> Request | None:
if self.paused:
return None
try:
return self.queue.pop()
except IndexError:
return None
def pause(self) -> None:
self.paused = True
def unpause(self) -> None:
self.paused = False
class MockSlot(NamedTuple):
active: list[Any]

13
tests/utils/__init__.py
View File

@ -1,13 +1,19 @@
from __future__ import annotations
import asyncio
import os
from collections.abc import Callable
from pathlib import Path
from typing import TYPE_CHECKING
from twisted.internet.defer import Deferred
from scrapy.settings import Settings, default_settings
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.defer import maybe_deferred_to_future
if TYPE_CHECKING:
from collections.abc import Callable
def twisted_sleep(seconds: float):
from twisted.internet import reactor
@ -48,3 +54,8 @@ class OneShotLoop:
def stop(self) -> None:
self.running = False
def assert_option_is_default(settings: Settings, key: str) -> None:
assert isinstance(settings, Settings)
assert settings[key] == getattr(default_settings, key)

34
tests/utils/base_commands.py Normal file
View File

@ -0,0 +1,34 @@
from __future__ import annotations
from shutil import copytree
from typing import TYPE_CHECKING
import pytest
from tests.utils.cmdline import call
if TYPE_CHECKING:
from pathlib import Path
class TestProjectBase:
"""A base class for tests that may need a Scrapy project."""
project_name = "testproject"
@pytest.fixture(scope="session")
def _proj_path_cached(self, tmp_path_factory: pytest.TempPathFactory) -> Path:
"""Create a Scrapy project in a temporary directory and return its path.
Used as a cache for ``proj_path``.
"""
tmp_path = tmp_path_factory.mktemp("proj")
call("startproject", self.project_name, cwd=tmp_path)
return tmp_path / self.project_name
@pytest.fixture
def proj_path(self, tmp_path: Path, _proj_path_cached: Path) -> Path:
"""Copy a pre-generated Scrapy project into a temporary directory and return its path."""
proj_path = tmp_path / self.project_name
copytree(_proj_path_cached, proj_path)
return proj_path

12
tests/utils/cmdline.py
View File

@ -2,12 +2,15 @@ from __future__ import annotations
import subprocess
import sys
from typing import Any
from typing import TYPE_CHECKING, Any
import pytest
from scrapy.utils.test import get_testenv
if TYPE_CHECKING:
from pathlib import Path
def call(*args: str, **popen_kwargs: Any) -> int:
args = (sys.executable, "-m", "scrapy.cmdline", *args)
@ -36,3 +39,10 @@ def proc(*args: str, **popen_kwargs: Any) -> tuple[int, str, str]:
pytest.fail("Command took too much time to complete")
return p.returncode, p.stdout, p.stderr
def write_recording_editor(editor: Path) -> None:
"""Create an executable editor script that writes the path it is asked to
open (its last argument) into the file given as its first argument."""
editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
editor.chmod(0o755)

29
tests/utils/feedexport.py Normal file
View File

@ -0,0 +1,29 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from urllib.parse import urljoin
from urllib.request import pathname2url
from scrapy import Field, Item
if TYPE_CHECKING:
from pathlib import Path
def path_to_url(path: str | Path) -> str:
return urljoin("file:", pathname2url(str(path)))
def printf_escape(s: str) -> str:
return s.replace("%", "%%")
class MyItem(Item):
foo = Field()
egg = Field()
baz = Field()
class MyItem2(Item):
foo = Field()
hello = Field()

40
tests/utils/redirect.py Normal file
View File

@ -0,0 +1,40 @@
from __future__ import annotations
from itertools import product
SCHEME_PARAMS = ("url", "location", "target")
HTTP_SCHEMES = ("http", "https")
NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo")
REDIRECT_SCHEME_CASES = (
# http/https → http/https redirects
*(
(
f"{input_scheme}://example.com/a",
f"{output_scheme}://example.com/b",
f"{output_scheme}://example.com/b",
)
for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2)
),
# http/https → data/file/ftp/s3/foo does not redirect
*(
(
f"{input_scheme}://example.com/a",
f"{output_scheme}://example.com/b",
None,
)
for input_scheme in HTTP_SCHEMES
for output_scheme in NON_HTTP_SCHEMES
),
# http/https → relative redirects
*(
(
f"{scheme}://example.com/a",
location,
f"{scheme}://example.com/b",
)
for scheme in HTTP_SCHEMES
for location in ("//example.com/b", "/b")
),
# Note: We do not test data/file/ftp/s3 schemes for the initial URL
# because their download handlers cannot return a status code of 3xx.
)

12
tests/utils/robotstxt.py Normal file
View File

@ -0,0 +1,12 @@
from __future__ import annotations
def rerp_available() -> bool:
# check if robotexclusionrulesparser is installed
try:
from robotexclusionrulesparser import ( # noqa: PLC0415
RobotExclusionRulesParser, # noqa: F401
)
except ImportError:
return False
return True