diff --git a/docs/news.rst b/docs/news.rst index aaeac347f..9830eda64 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -15,6 +15,35 @@ Backward-incompatible changes ``True`` when running Scrapy via :ref:`its command-line tool ` to avoid a reactor mismatch exception. +- The classes listed below are now :term:`abstract base classes `. They cannot be instantiated directly and their subclasses + need to override the abstract methods listed below to be able to be + instantiated. If you previously instantiated these classes directly, you + will now need to subclass them and provide trivial (e.g. empty) + implementations for the abstract methods. + + - :class:`scrapy.commands.ScrapyCommand` + + - :meth:`~scrapy.commands.ScrapyCommand.run` + + - :meth:`~scrapy.commands.ScrapyCommand.short_desc` + + - :class:`scrapy.exporters.BaseItemExporter` + + - :meth:`~scrapy.exporters.BaseItemExporter.export_item` + + - :class:`scrapy.extensions.feedexport.BlockingFeedStorage` + + - :meth:`~scrapy.extensions.feedexport.BlockingFeedStorage._store_in_thread` + + - :class:`scrapy.middleware.MiddlewareManager` + + - :meth:`~scrapy.middleware.MiddlewareManager._get_mwlist_from_settings` + + - :class:`scrapy.spidermiddlewares.referer.ReferrerPolicy` + + - :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer` + .. _release-2.13.2: diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 67178b8fd..2eb59fe44 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -354,7 +354,7 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'`` Acceptable values for REFERRER_POLICY ************************************* -- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` +- either a path to a :class:`scrapy.spidermiddlewares.referer.ReferrerPolicy` subclass — a custom policy or one of the built-in ones (see classes below), - or one or more comma-separated standard W3C-defined string values, - or the special ``"scrapy-default"``. @@ -373,6 +373,8 @@ String value Class name (as a string) `"unsafe-url"`_ :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy` ======================================= ======================================================================== +.. autoclass:: ReferrerPolicy + .. autoclass:: DefaultReferrerPolicy .. warning:: Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_, diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py index 4ce070e6e..d5945f6f5 100644 --- a/scrapy/commands/__init__.py +++ b/scrapy/commands/__init__.py @@ -7,6 +7,7 @@ from __future__ import annotations import argparse import builtins import os +from abc import ABC, abstractmethod from pathlib import Path from typing import TYPE_CHECKING, Any @@ -22,7 +23,7 @@ if TYPE_CHECKING: from scrapy.settings import Settings -class ScrapyCommand: +class ScrapyCommand(ABC): requires_project: bool = False requires_crawler_process: bool = True crawler_process: CrawlerProcessBase | None = None # set in scrapy.cmdline @@ -46,6 +47,7 @@ class ScrapyCommand: """ return "" + @abstractmethod def short_desc(self) -> str: """ A short description of the command @@ -128,6 +130,7 @@ class ScrapyCommand: if opts.pdb: failure.startDebugMode() + @abstractmethod def run(self, args: list[str], opts: argparse.Namespace) -> None: """ Entry point for running commands diff --git a/scrapy/exporters.py b/scrapy/exporters.py index 0a641752e..e18f1e6ed 100644 --- a/scrapy/exporters.py +++ b/scrapy/exporters.py @@ -8,6 +8,7 @@ import csv import marshal import pickle import pprint +from abc import ABC, abstractmethod from collections.abc import Callable, Iterable, Mapping from io import BytesIO, TextIOWrapper from typing import TYPE_CHECKING, Any @@ -35,7 +36,7 @@ __all__ = [ ] -class BaseItemExporter: +class BaseItemExporter(ABC): def __init__(self, *, dont_fail: bool = False, **kwargs: Any): self._kwargs: dict[str, Any] = kwargs self._configure(kwargs, dont_fail=dont_fail) @@ -54,6 +55,7 @@ class BaseItemExporter: if not dont_fail and options: raise TypeError(f"Unexpected options: {', '.join(options.keys())}") + @abstractmethod def export_item(self, item: Any) -> None: raise NotImplementedError @@ -63,10 +65,10 @@ class BaseItemExporter: serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x) return serializer(value) - def start_exporting(self) -> None: + def start_exporting(self) -> None: # noqa: B027 pass - def finish_exporting(self) -> None: + def finish_exporting(self) -> None: # noqa: B027 pass def _get_serialized_fields( diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index f7bf50a5c..557ad99e1 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -11,6 +11,7 @@ import logging import re import sys import warnings +from abc import ABC, abstractmethod from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path, PureWindowsPath @@ -140,7 +141,7 @@ class FeedStorageProtocol(Protocol): @implementer(IFeedStorage) -class BlockingFeedStorage: +class BlockingFeedStorage(ABC): def open(self, spider: Spider) -> IO[bytes]: path = spider.crawler.settings["FEED_TEMPDIR"] if path and not Path(path).is_dir(): @@ -151,6 +152,7 @@ class BlockingFeedStorage: def store(self, file: IO[bytes]) -> Deferred[None] | None: return deferToThread(self._store_in_thread, file) + @abstractmethod def _store_in_thread(self, file: IO[bytes]) -> None: raise NotImplementedError diff --git a/scrapy/middleware.py b/scrapy/middleware.py index 2b67dcd21..25f2aa199 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -3,6 +3,7 @@ from __future__ import annotations import logging import pprint import warnings +from abc import ABC, abstractmethod from collections import defaultdict, deque from typing import TYPE_CHECKING, Any, TypeVar, cast @@ -32,10 +33,10 @@ _T = TypeVar("_T") _T2 = TypeVar("_T2") -class MiddlewareManager: +class MiddlewareManager(ABC): """Base class for implementing middleware managers""" - component_name = "foo middleware" + component_name: str def __init__(self, *middlewares: Any) -> None: self.middlewares = middlewares @@ -48,6 +49,7 @@ class MiddlewareManager: self._add_middleware(mw) @classmethod + @abstractmethod def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]: raise NotImplementedError diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py index f5d406c13..1d4d1527b 100644 --- a/scrapy/spidermiddlewares/referer.py +++ b/scrapy/spidermiddlewares/referer.py @@ -6,6 +6,7 @@ originated it. from __future__ import annotations import warnings +from abc import ABC, abstractmethod from typing import TYPE_CHECKING, cast from urllib.parse import urlparse @@ -45,10 +46,13 @@ POLICY_UNSAFE_URL = "unsafe-url" POLICY_SCRAPY_DEFAULT = "scrapy-default" -class ReferrerPolicy: +class ReferrerPolicy(ABC): + """Abstract base class for referrer policies.""" + NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES name: str + @abstractmethod def referrer(self, response_url: str, request_url: str) -> str | None: raise NotImplementedError diff --git a/tests/test_commands.py b/tests/test_commands.py index 851c92db4..1b36fbb31 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -25,9 +25,17 @@ if TYPE_CHECKING: import os +class EmptyCommand(ScrapyCommand): + def short_desc(self) -> str: + return "" + + def run(self, args: list[str], opts: argparse.Namespace) -> None: + pass + + class TestCommandSettings: def setup_method(self): - self.command = ScrapyCommand() + self.command = EmptyCommand() self.command.settings = Settings() self.parser = argparse.ArgumentParser( formatter_class=ScrapyHelpFormatter, conflict_handler="resolve" diff --git a/tests/test_exporters.py b/tests/test_exporters.py index 05e8865bc..5719d5bb0 100644 --- a/tests/test_exporters.py +++ b/tests/test_exporters.py @@ -4,6 +4,7 @@ import marshal import pickle import re import tempfile +from abc import ABC, abstractmethod from datetime import datetime from io import BytesIO from typing import Any @@ -53,7 +54,7 @@ class CustomFieldDataclass: age: int = dataclasses.field(metadata={"serializer": custom_serializer}) -class TestBaseItemExporter: +class TestBaseItemExporter(ABC): item_class: type = MyItem custom_field_item_class: type = CustomFieldItem @@ -62,10 +63,11 @@ class TestBaseItemExporter: self.output = BytesIO() self.ie = self._get_exporter() - def _get_exporter(self, **kwargs): - return BaseItemExporter(**kwargs) + @abstractmethod + def _get_exporter(self, **kwargs) -> BaseItemExporter: + raise NotImplementedError - def _check_output(self): + def _check_output(self): # noqa: B027 pass def _assert_expected_item(self, exported_dict): @@ -83,11 +85,7 @@ class TestBaseItemExporter: def assertItemExportWorks(self, item): self.ie.start_exporting() - try: - self.ie.export_item(item) - except NotImplementedError: - if self.ie.__class__ is not BaseItemExporter: - raise + self.ie.export_item(item) self.ie.finish_exporting() # Delete the item exporter object, so that if it causes the output # file handle to be closed, which should not be the case, follow-up @@ -132,11 +130,6 @@ class TestBaseItemExporter: assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == "24" -class TestBaseItemExporterDataclass(TestBaseItemExporter): - item_class = MyDataClass - custom_field_item_class = CustomFieldDataclass - - class TestPythonItemExporter(TestBaseItemExporter): def _get_exporter(self, **kwargs): return PythonItemExporter(**kwargs) @@ -670,6 +663,9 @@ class TestCustomExporterItem: return str(int(value) + 1) return super().serialize_field(field, name, value) + def export_item(self, item: Any) -> None: + pass + i = self.item_class(name="John", age="22") a = ItemAdapter(i) ie = CustomItemExporter() diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 7073d5a35..ce1b7952e 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -18,7 +18,7 @@ from io import BytesIO from logging import getLogger from pathlib import Path from string import ascii_letters, digits -from typing import TYPE_CHECKING, Any +from typing import IO, TYPE_CHECKING, Any from unittest import mock from urllib.parse import quote, urljoin from urllib.request import pathname2url @@ -236,6 +236,11 @@ class TestFTPFeedStorage(unittest.TestCase): assert st.password == string.punctuation +class MyBlockingFeedStorage(BlockingFeedStorage): + def _store_in_thread(self, file: IO[bytes]) -> None: + return + + class TestBlockingFeedStorage: def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): @@ -245,14 +250,14 @@ class TestBlockingFeedStorage: return TestSpider.from_crawler(crawler) def test_default_temp_dir(self): - b = BlockingFeedStorage() + b = MyBlockingFeedStorage() storage_file = b.open(self.get_test_spider()) storage_dir = Path(storage_file.name).parent assert str(storage_dir) == tempfile.gettempdir() def test_temp_file(self, tmp_path): - b = BlockingFeedStorage() + b = MyBlockingFeedStorage() spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)}) storage_file = b.open(spider) @@ -260,7 +265,7 @@ class TestBlockingFeedStorage: assert storage_dir == tmp_path def test_invalid_folder(self, tmp_path): - b = BlockingFeedStorage() + b = MyBlockingFeedStorage() invalid_path = tmp_path / "invalid_path" spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)}) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index d004d4d93..3dd1334b2 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -39,6 +39,8 @@ class MOff: class MyMiddlewareManager(MiddlewareManager): + component_name = "my" + @classmethod def _get_mwlist_from_settings(cls, settings): return [M1, MOff, M3] @@ -65,7 +67,7 @@ class TestMiddlewareManager: def test_enabled(self): m1, m2, m3 = M1(), M2(), M3() - mwman = MiddlewareManager(m1, m2, m3) + mwman = MyMiddlewareManager(m1, m2, m3) assert mwman.middlewares == (m1, m2, m3) def test_enabled_from_settings(self):