diff --git a/conftest.py b/conftest.py index 5a535c168..ad11cedcc 100644 --- a/conftest.py +++ b/conftest.py @@ -47,7 +47,7 @@ if not H2_ENABLED: collect_ignore.extend( ( "scrapy/core/downloader/handlers/http2.py", - *_py_files("scrapy/core/http2"), + *_py_files("scrapy/core/_http2"), ) ) @@ -107,6 +107,14 @@ def pytest_configure(config): install_reactor_import_hook() +def pytest_collection_modifyitems(items): + for item in items: + if item.get_closest_marker("requires_internet"): + # Requests to real websites fail every now and then in CI for + # reasons unrelated to the code under test. + item.add_marker(pytest.mark.flaky(reruns=2, reruns_delay=5)) + + def pytest_runtest_setup(item): # Skip tests based on reactor markers reactor = item.config.getoption("--reactor") diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 433a6d139..d95e7b52b 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -187,12 +187,6 @@ If you want to use this handler you need to replace the default one for the Features and limitations ^^^^^^^^^^^^^^^^^^^^^^^^ -.. warning:: - - This handler is experimental, and not yet recommended for production - environments. Future Scrapy versions may introduce related changes without - a deprecation period or warning. - =========================== ================================================ HTTP proxies No (not implemented) SOCKS proxies No (not supported by the library) diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index cb803abb6..42314f9a9 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1582,6 +1582,20 @@ Default: ``{}`` A dict containing the pipelines enabled by default in Scrapy. You should never modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead. +.. setting:: ITEM_PROCESSOR + +ITEM_PROCESSOR +-------------- + +Default: ``"scrapy.pipelines.ItemPipelineManager"`` + +The :ref:`component ` that builds the :ref:`item pipeline +` from :setting:`ITEM_PIPELINES` and runs scraped items +through it. It must implement :class:`~scrapy.pipelines.ItemProcessorProtocol`. + +.. autoclass:: scrapy.pipelines.ItemProcessorProtocol + :members: + .. setting:: JOBDIR diff --git a/scrapy/core/http2/__init__.py b/scrapy/core/_http2/__init__.py similarity index 100% rename from scrapy/core/http2/__init__.py rename to scrapy/core/_http2/__init__.py diff --git a/scrapy/core/http2/agent.py b/scrapy/core/_http2/agent.py similarity index 99% rename from scrapy/core/http2/agent.py rename to scrapy/core/_http2/agent.py index 8850a512d..ce601c52b 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/_http2/agent.py @@ -9,8 +9,8 @@ from twisted.python.failure import Failure from twisted.web.client import URI, BrowserLikePolicyForHTTPS, _StandardEndpointFactory from twisted.web.error import SchemeNotSupported +from scrapy.core._http2.protocol import H2ClientFactory, H2ClientProtocol from scrapy.core.downloader.contextfactory import _AcceptableProtocolsContextFactory -from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol if TYPE_CHECKING: from twisted.internet.base import ReactorBase diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/_http2/protocol.py similarity index 99% rename from scrapy/core/http2/protocol.py rename to scrapy/core/_http2/protocol.py index a4bc7bf8d..db89d066f 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/_http2/protocol.py @@ -31,7 +31,7 @@ from twisted.internet.ssl import Certificate from twisted.protocols.policies import TimeoutMixin from zope.interface import implementer -from scrapy.core.http2.stream import Stream, StreamCloseReason +from scrapy.core._http2.stream import Stream, StreamCloseReason from scrapy.exceptions import DownloadTimeoutError from scrapy.http import Request, Response from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute diff --git a/scrapy/core/http2/stream.py b/scrapy/core/_http2/stream.py similarity index 99% rename from scrapy/core/http2/stream.py rename to scrapy/core/_http2/stream.py index 4fc300d90..4a07d198b 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/_http2/stream.py @@ -27,7 +27,7 @@ from scrapy.utils.httpobj import urlparse_cached if TYPE_CHECKING: from collections.abc import Sequence - from scrapy.core.http2.protocol import H2ClientProtocol + from scrapy.core._http2.protocol import H2ClientProtocol from scrapy.crawler import Crawler from scrapy.http import Request, Response diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index 66e62c55a..e159d2dac 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -4,9 +4,9 @@ from time import monotonic from typing import TYPE_CHECKING from urllib.parse import urldefrag +from scrapy.core._http2.agent import H2Agent, H2ConnectionPool from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings from scrapy.core.downloader.handlers._base_http import BaseHttpDownloadHandler -from scrapy.core.http2.agent import H2Agent, H2ConnectionPool from scrapy.exceptions import ( DownloadTimeoutError, NotConfigured, diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py index adffbcbc4..1b66eae4a 100644 --- a/scrapy/extensions/periodic_log.py +++ b/scrapy/extensions/periodic_log.py @@ -1,11 +1,12 @@ from __future__ import annotations import logging +import warnings from datetime import datetime, timezone from typing import TYPE_CHECKING, Any from scrapy import Spider, signals -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call from scrapy.utils.serialize import ScrapyJSONEncoder @@ -38,6 +39,7 @@ class PeriodicLog: ): self.stats: StatsCollector = stats self.interval: float = interval + self._multiplier: float = 60.0 / interval self.task: AsyncioLoopingCall | LoopingCall | None = None self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4) self.ext_stats_enabled: bool = bool(ext_stats) @@ -56,6 +58,24 @@ class PeriodicLog: ) self.ext_timing_enabled: bool = ext_timing_enabled + @property + def multiplier(self) -> float: + warnings.warn( + "The PeriodicLog.multiplier attribute is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return self._multiplier + + @multiplier.setter + def multiplier(self, value: float) -> None: + warnings.warn( + "The PeriodicLog.multiplier attribute is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + self._multiplier = value + @classmethod def from_crawler(cls, crawler: Crawler) -> Self: interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL") diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py index 383c461e6..e6483d939 100644 --- a/scrapy/pipelines/__init__.py +++ b/scrapy/pipelines/__init__.py @@ -8,7 +8,7 @@ from __future__ import annotations import asyncio import warnings -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any, Protocol, cast from twisted.internet.defer import Deferred, DeferredList, FirstError @@ -28,6 +28,23 @@ if TYPE_CHECKING: from scrapy.settings import Settings +class ItemProcessorProtocol(Protocol): + """Protocol for item processor implementations. + + See :setting:`ITEM_PROCESSOR`. + """ + + async def open_spider_async(self) -> None: + """Get the item processor ready to process items.""" + + async def process_item_async(self, item: Any) -> Any: + """Return the processed *item*, or raise + :exc:`~scrapy.exceptions.DropItem` to drop it.""" + + async def close_spider_async(self) -> None: + """Release any resource that the item processor is using.""" + + class ItemPipelineManager(MiddlewareManager): component_name = "item pipeline" diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index e666f4ddd..9d67e98bf 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -27,9 +27,7 @@ from twisted.internet.defer import Deferred, maybeDeferred from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.media import ( - FileException as FileException, # noqa: PLC0414 # re-exported for backward compatibility -) +from scrapy.pipelines.media import FileException as _FileException from scrapy.pipelines.media import ( FileInfo, FileInfoOrError, @@ -626,7 +624,7 @@ class FilesPipeline(MediaPipeline): f"{request} referred in <{referer}>: {failure.value}", extra={"spider": info.spider}, ) - raise FileException + raise _FileException async def media_downloaded( self, @@ -645,7 +643,7 @@ class FilesPipeline(MediaPipeline): {"status": response.status, "request": request, "referer": referer}, extra={"spider": info.spider}, ) - raise FileException("download-error") + raise _FileException("download-error") if not response.body: logger.warning( @@ -654,7 +652,7 @@ class FilesPipeline(MediaPipeline): {"request": request, "referer": referer}, extra={"spider": info.spider}, ) - raise FileException("empty-content") + raise _FileException("empty-content") status = "cached" if "cached" in response.flags else "downloaded" logger.debug( @@ -670,7 +668,7 @@ class FilesPipeline(MediaPipeline): checksum: str = await ensure_awaitable( self.file_downloaded(response, request, info, item=item) ) - except FileException as exc: + except _FileException as exc: logger.warning( "File (error): Error processing file from %(request)s " "referred in <%(referer)s>: %(errormsg)s", @@ -687,7 +685,7 @@ class FilesPipeline(MediaPipeline): exc_info=True, extra={"spider": info.spider}, ) - raise FileException(str(exc)) from exc + raise _FileException(str(exc)) from exc return { "url": request.url, @@ -770,3 +768,15 @@ class FilesPipeline(MediaPipeline): if media_type: media_ext = cast("str", mimetypes.guess_extension(media_type)) return f"full/{media_guid}{media_ext}" + + +def __getattr__(name: str) -> Any: + if name == "FileException": + warnings.warn( + "scrapy.pipelines.files.FileException is deprecated, use " + "scrapy.pipelines.media.FileException instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _FileException + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index 5e7a4b409..7186fc8de 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -18,13 +18,8 @@ from itemadapter import ItemAdapter from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK -from scrapy.pipelines.files import ( - FileException, - FilesPipeline, - GCSFilesStore, - S3FilesStore, - _md5sum, -) +from scrapy.pipelines.files import FilesPipeline, GCSFilesStore, S3FilesStore, _md5sum +from scrapy.pipelines.media import FileException from scrapy.utils.defer import ensure_awaitable from scrapy.utils.python import to_bytes diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 26d051e79..258ede04e 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -146,7 +146,7 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase): response = await download_handler.download_request(request) assert response.text == actual_content_length assert ( - "scrapy.core.http2.stream", + "scrapy.core._http2.stream", logging.WARNING, f"Ignoring bad Content-Length header " f"{bad_content_length!r} of request {request}, sending " diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 340f8e28e..fd7e3958d 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -7,7 +7,7 @@ from typing import TYPE_CHECKING, Any import pytest -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.periodic_log import PeriodicLog from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler @@ -249,3 +249,17 @@ class TestPeriodicLog: assert data["time"]["log_interval_real"] >= 0 assert data["time"]["elapsed"] >= 0 assert data["time"]["start_time"] <= data["time"]["utcnow"] + + def test_multiplier_deprecated(self) -> None: + crawler = get_crawler( + MetaSpider, + {"PERIODIC_LOG_TIMING_ENABLED": True, "LOGSTATS_INTERVAL": 30}, + ) + crawler._apply_settings() + ext = build_from_crawler(PeriodicLog, crawler) + with pytest.warns(ScrapyDeprecationWarning): + assert ext.multiplier == 2.0 + with pytest.warns(ScrapyDeprecationWarning): + ext.multiplier = 3.0 + with pytest.warns(ScrapyDeprecationWarning): + assert ext.multiplier == 3.0 diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 0b675fdb6..aa52b873e 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -36,7 +36,7 @@ from tests.mockserver.utils import ssl_context_factory if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable, Coroutine, Generator - from scrapy.core.http2.protocol import H2ClientProtocol + from scrapy.core._http2.protocol import H2ClientProtocol pytestmark = [ @@ -242,7 +242,7 @@ class TestHttps2ClientProtocol: ) -> AsyncGenerator[H2ClientProtocol]: from twisted.internet import reactor - from scrapy.core.http2.protocol import H2ClientFactory # noqa: PLC0415 + from scrapy.core._http2.protocol import H2ClientFactory # noqa: PLC0415 client_options = optionsForClientTLS( hostname=self.host, @@ -460,7 +460,7 @@ class TestHttps2ClientProtocol: def test_invalid_negotiated_protocol( self, server_port: int, client: H2ClientProtocol ) -> Generator[Deferred[Any], Any, None]: - with mock.patch("scrapy.core.http2.protocol.PROTOCOL_NAME", new=b"not-h2"): + with mock.patch("scrapy.core._http2.protocol.PROTOCOL_NAME", new=b"not-h2"): request = Request(url=self.get_url(server_port, "/status?n=200")) with pytest.raises(ResponseFailed): yield make_request_dfd(client, request) @@ -527,7 +527,7 @@ class TestHttps2ClientProtocol: expected_body: bytes, caplog: pytest.LogCaptureFixture, ) -> None: - with caplog.at_level("WARNING", "scrapy.core.http2.stream"): + with caplog.at_level("WARNING", "scrapy.core._http2.stream"): response = await make_request(client, request) assert response.status == 200 assert response.body == expected_body @@ -605,7 +605,7 @@ class TestHttps2ClientProtocol: def assert_inactive_stream(failure): assert failure.check(ResponseFailed) is not None - from scrapy.core.http2.stream import InactiveStreamClosed # noqa: PLC0415 + from scrapy.core._http2.stream import InactiveStreamClosed # noqa: PLC0415 assert any( isinstance(e, InactiveStreamClosed) for e in failure.value.reasons @@ -692,7 +692,7 @@ class TestHttps2ClientProtocol: @staticmethod async def _check_invalid_netloc(client: H2ClientProtocol, url: str) -> None: - from scrapy.core.http2.stream import InvalidHostname # noqa: PLC0415 + from scrapy.core._http2.stream import InvalidHostname # noqa: PLC0415 request = Request(url) with pytest.raises(InvalidHostname) as exc_info: diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 53f998762..928f4e2c2 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -24,18 +24,18 @@ from twisted.internet.defer import Deferred from twisted.python.failure import Failure from scrapy.crawler import Crawler -from scrapy.exceptions import IgnoreRequest, NotConfigured +from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request, Response from scrapy.item import Field, Item +from scrapy.pipelines import files from scrapy.pipelines.files import ( - FileException, FilesPipeline, FSFilesStore, FTPFilesStore, GCSFilesStore, S3FilesStore, ) -from scrapy.pipelines.media import _MediaRequestFiltered +from scrapy.pipelines.media import FileException, _MediaRequestFiltered from scrapy.settings import Settings from scrapy.utils.asyncio import call_later from scrapy.utils.defer import maybe_deferred_to_future @@ -1249,3 +1249,11 @@ def test_files_pipeline_raises_notconfigured_when_files_store_invalid(store): with pytest.raises(NotConfigured): build_from_crawler(FilesPipeline, crawler) + + +def test_file_exception_deprecated_import(): + with pytest.warns(ScrapyDeprecationWarning, match="FileException"): + assert files.FileException is FileException + + with pytest.raises(AttributeError): + files.nonexistent diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 305837f57..231f6748f 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -10,8 +10,8 @@ from twisted.python.failure import Failure from scrapy import signals from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import Request, Response -from scrapy.pipelines.files import FileException from scrapy.pipelines.media import ( + FileException, FileInfo, FileInfoOrError, MediaPipeline, diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py index 8cd0458cf..a01f74658 100644 --- a/tests/utils/bases/download_handlers_http.py +++ b/tests/utils/bases/download_handlers_http.py @@ -1577,6 +1577,12 @@ class TestMitmProxyBase(ABC): assert "Proxy Authentication Required" in log or "407" in log +# Tests below are rerun on failure (see pytest_collection_modifyitems() in the +# root conftest.py), so an attempt must give up soon enough for a rerun to be +# cheap. +REAL_WEBSITE_SETTINGS = {"DOWNLOAD_TIMEOUT": 30} + + class TestRealWebsiteBase(ABC): @property @abstractmethod @@ -1601,7 +1607,9 @@ class TestRealWebsiteBase(ABC): async def get_dh( self, settings_dict: dict[str, Any] | None = None ) -> AsyncGenerator[DownloadHandlerProtocol]: - crawler = get_crawler(DefaultSpider, settings_dict) + crawler = get_crawler( + DefaultSpider, {**REAL_WEBSITE_SETTINGS, **(settings_dict or {})} + ) crawler.spider = crawler._create_spider() dh = build_from_crawler(self.download_handler_cls, crawler) try: @@ -1619,7 +1627,9 @@ class TestRealWebsiteBase(ABC): @coroutine_test async def test_download_with_spider(self) -> None: - crawler = get_crawler(SingleRequestSpider, self.settings_dict) + crawler = get_crawler( + SingleRequestSpider, {**REAL_WEBSITE_SETTINGS, **(self.settings_dict or {})} + ) await maybe_deferred_to_future( crawler.crawl(seed=Request("https://books.toscrape.com/")) ) diff --git a/tox.ini b/tox.ini index 03fee6dd3..34af7bac3 100644 --- a/tox.ini +++ b/tox.ini @@ -44,6 +44,7 @@ deps = pygments pytest pytest-cov >= 7.0.0 + pytest-rerunfailures pytest-timeout pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 @@ -124,7 +125,7 @@ commands = [testenv:twinecheck] basepython = python3 deps = - twine==6.2.0 + twine==7.0.0 build==1.5.0 commands = python -m build --sdist