From 10089c6fe2028b879f9f60e9598fa580ef6a3e33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 18 Nov 2024 09:07:32 +0100 Subject: [PATCH] 2.12 release notes (#6226) * Cover 2.12 in the release notes up to 9bb973dc54766a0f8d10eca0947d11f195c1a1be * Add one more highlight * Better merge of the news entries. * Cover 2.12 in the release notes up to 642af40. * Cover 2.12 in the release notes up to 7a0a34b. * Cover 2.12 in the release notes up to b4bad97. * Add not yet merged PRs #6463, #6507, #6511 to the 2.12 release notes. * Cover 2.12 in the release notes up to d85c39f, small fixes. * Cover 2.12 in the release notes up to d215669. * Cover #6527 in the release notes. * Address PR feedback. * Cover recent PRs. * Finalize the 2.12.0 release notes, small additional fixes. --------- Co-authored-by: Andrey Rakhmatullin --- docs/news.rst | 550 +++++++++++++++++++++++++- docs/topics/addons.rst | 7 +- docs/topics/api.rst | 4 +- docs/topics/components.rst | 16 +- docs/topics/downloader-middleware.rst | 4 - docs/topics/spider-middleware.rst | 2 +- scrapy/crawler.py | 42 ++ scrapy/extensions/feedexport.py | 6 +- scrapy/pipelines/media.py | 14 +- scrapy/utils/misc.py | 2 + scrapy/utils/python.py | 3 +- tests/test_pipeline_media.py | 29 +- 12 files changed, 647 insertions(+), 32 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 3c9e58cca..025eb09ba 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,23 +3,555 @@ Release notes ============= -.. _release-VERSION: +.. _release-2.12.0: -Scrapy VERSION (YYYY-MM-DD) ---------------------------- +Scrapy 2.12.0 (2024-11-18) +-------------------------- -New features -~~~~~~~~~~~~ +Highlights: -- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``, - ``SpiderLoader`` does not raise :exc:`SyntaxError` but emits a warning instead. +- Dropped support for Python 3.8, added support for Python 3.13 + +- :meth:`~scrapy.Spider.start_requests` can now yield items + +- Added :class:`~scrapy.http.JsonResponse` + +- Added :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM` + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for Python 3.8. + (:issue:`6466`, :issue:`6472`) + +- Added support for Python 3.13. + (:issue:`6166`) + +- Minimum versions increased for these dependencies: + + - Twisted_: 18.9.0 → 21.7.0 + + - cryptography_: 36.0.0 → 37.0.0 + + - pyOpenSSL_: 21.0.0 → 22.0.0 + + - lxml_: 4.4.1 → 4.6.0 + +- Removed ``setuptools`` from the dependency list. + (:issue:`6487`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- User-defined cookies for HTTPS requests will have the ``secure`` flag set + to ``True`` unless it's set to ``False`` explictly. This is important when + these cookies are reused in HTTP requests, e.g. after a redirect to an HTTP + URL. + (:issue:`6357`) + +- The Reppy-based ``robots.txt`` parser, + ``scrapy.robotstxt.ReppyRobotParser``, was removed, as it doesn't support + Python 3.9+. + (:issue:`5230`, :issue:`6099`, :issue:`6499`) + +- The initialization API of :class:`scrapy.pipelines.media.MediaPipeline` and + its subclasses was improved and it's possible that some previously working + usage scenarios will no longer work. It can only affect you if you define + custom subclasses of ``MediaPipeline`` or create instances of these + pipelines via ``from_settings()`` or ``__init__()`` calls instead of + ``from_crawler()`` calls. + + Previously, ``MediaPipeline.from_crawler()`` called the ``from_settings()`` + method if it existed or the ``__init__()`` method otherwise, and then did + some additional initialization using the ``crawler`` instance. If the + ``from_settings()`` method existed (like in ``FilesPipeline``) it called + ``__init__()`` to create the instance. It wasn't possible to override + ``from_crawler()`` without calling ``MediaPipeline.from_crawler()`` from it + which, in turn, couldn't be called in some cases (including subclasses of + ``FilesPipeline``). + + Now, in line with the general usage of ``from_crawler()`` and + ``from_settings()`` and the deprecation of the latter the recommended + initialization order is the following one: + + - All ``__init__()`` methods should take a ``crawler`` argument. If they + also take a ``settings`` argument they should ignore it, using + ``crawler.settings`` instead. When they call ``__init__()`` of the base + class they should pass the ``crawler`` argument to it too. + - A ``from_settings()`` method shouldn't be defined. Class-specific + initialization code should go into either an overriden ``from_crawler()`` + method or into ``__init__()``. + - It's now possible to override ``from_crawler()`` and it's not necessary + to call ``MediaPipeline.from_crawler()`` in it if other recommendations + were followed. + - If pipeline instances were created with ``from_settings()`` or + ``__init__()`` calls (which wasn't supported even before, as it missed + important initialization code), they should now be created with + ``from_crawler()`` calls. + + (:issue:`6540`) + +- The ``response_body`` argument of :meth:`ImagesPipeline.convert_image + ` is now + positional-only, as it was changed from optional to required. + (:issue:`6500`) + +- The ``convert`` argument of :func:`scrapy.utils.conf.build_component_list` + is now positional-only, as the preceding argument (``custom``) was removed. + (:issue:`6500`) + +- The ``overwrite_output`` argument of + :func:`scrapy.utils.conf.feed_process_params_from_cli` is now + positional-only, as the preceding argument (``output_format``) was removed. + (:issue:`6500`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed the ``scrapy.utils.request.request_fingerprint()`` function, + deprecated in Scrapy 2.7.0. + (:issue:`6212`, :issue:`6213`) + +- Removed support for value ``"2.6"`` of setting + ``REQUEST_FINGERPRINTER_IMPLEMENTATION``, deprecated in Scrapy 2.7.0. + (:issue:`6212`, :issue:`6213`) + +- :class:`~scrapy.dupefilters.RFPDupeFilter` subclasses now require + supporting the ``fingerprinter`` parameter in their ``__init__`` method, + introduced in Scrapy 2.7.0. + (:issue:`6102`, :issue:`6113`) + +- Removed the ``scrapy.downloadermiddlewares.decompression`` module, + deprecated in Scrapy 2.7.0. + (:issue:`6100`, :issue:`6113`) + +- Removed the ``scrapy.utils.response.response_httprepr()`` function, + deprecated in Scrapy 2.6.0. + (:issue:`6111`, :issue:`6116`) + +- Spiders with spider-level HTTP authentication, i.e. with the ``http_user`` + or ``http_pass`` attributes, must now define ``http_auth_domain`` as well, + which was introduced in Scrapy 2.5.1. + (:issue:`6103`, :issue:`6113`) + +- :ref:`Media pipelines ` methods ``file_path()``, + ``file_downloaded()``, ``get_images()``, ``image_downloaded()``, + ``media_downloaded()``, ``media_to_download()``, and ``thumb_path()`` must + now support an ``item`` parameter, added in Scrapy 2.4.0. + (:issue:`6107`, :issue:`6113`) + +- The ``__init__()`` and ``from_crawler()`` methods of :ref:`feed storage + backend classes ` must now support the keyword-only + ``feed_options`` parameter, introduced in Scrapy 2.4.0. + (:issue:`6105`, :issue:`6113`) + +- Removed the ``scrapy.loader.common`` and ``scrapy.loader.processors`` + modules, deprecated in Scrapy 2.3.0. + (:issue:`6106`, :issue:`6113`) + +- Removed the ``scrapy.utils.misc.extract_regex()`` function, deprecated in + Scrapy 2.3.0. + (:issue:`6106`, :issue:`6113`) + +- Removed the ``scrapy.http.JSONRequest`` class, replaced with + ``JsonRequest`` in Scrapy 1.8.0. + (:issue:`6110`, :issue:`6113`) + +- ``scrapy.utils.log.logformatter_adapter`` no longer supports missing + ``args``, ``level``, or ``msg`` parameters, and no longer supports a + ``format`` parameter, all scenarios that were deprecated in Scrapy 1.0.0. + (:issue:`6109`, :issue:`6116`) + +- A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that + does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface + will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at + run time. Non-compliant classes have been triggering a deprecation warning + since Scrapy 1.0.0. + (:issue:`6101`, :issue:`6113`) + +- Removed the ``--output-format``/``-t`` command line option, deprecated in + Scrapy 2.1.0. ``-O :`` should be used instead. + (:issue:`6500`) + +- Running :meth:`~scrapy.crawler.Crawler.crawl` more than once on the same + :class:`~scrapy.crawler.Crawler` instance, deprecated in Scrapy 2.11.0, now + raises an exception. + (:issue:`6500`) + +- Subclassing + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + without support for the ``crawler`` argument in ``__init__()`` and without + a custom ``from_crawler()`` method, deprecated in Scrapy 2.5.0, is no + longer allowed. + (:issue:`6500`) + +- Removed the ``EXCEPTIONS_TO_RETRY`` attribute of + :class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware`, deprecated in + Scrapy 2.10.0. + (:issue:`6500`) + +- Removed support for :ref:`S3 feed exports ` without + the boto3_ package installed, deprecated in Scrapy 2.10.0. + (:issue:`6500`) + +- Removed the ``scrapy.extensions.feedexport._FeedSlot`` class, deprecated in + Scrapy 2.10.0. + (:issue:`6500`) + +- Removed the ``scrapy.pipelines.images.NoimagesDrop`` exception, deprecated + in Scrapy 2.8.0. + (:issue:`6500`) + +- The ``response_body`` argument of :meth:`ImagesPipeline.convert_image + ` is now required, + not passing it was deprecated in Scrapy 2.8.0. + (:issue:`6500`) + +- Removed the ``custom`` argument of + :func:`scrapy.utils.conf.build_component_list`, deprecated in Scrapy + 2.10.0. + (:issue:`6500`) + +- Removed the ``scrapy.utils.reactor.get_asyncio_event_loop_policy()`` + function, deprecated in Scrapy 2.9.0. Use :func:`asyncio.get_event_loop` + and related standard library functions instead. + (:issue:`6500`) Deprecations ~~~~~~~~~~~~ -- :meth:`scrapy.core.downloader.Downloader._get_slot_key` is deprecated, use +- The ``from_settings()`` methods of the :ref:`Scrapy components + ` that have them are now deprecated. ``from_crawler()`` + should now be used instead. Affected components: + + - :class:`scrapy.dupefilters.RFPDupeFilter` + - :class:`scrapy.mail.MailSender` + - :class:`scrapy.middleware.MiddlewareManager` + - :class:`scrapy.core.downloader.contextfactory.ScrapyClientContextFactory` + - :class:`scrapy.pipelines.files.FilesPipeline` + - :class:`scrapy.pipelines.images.ImagesPipeline` + - :class:`scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` + + (:issue:`6540`) + +- It's now deprecated to have a ``from_settings()`` method but no + ``from_crawler()`` method in 3rd-party :ref:`Scrapy components + `. You can define a simple ``from_crawler()`` method + that calls ``cls.from_settings(crawler.settings)`` to fix this if you don't + want to refactor the code. Note that if you have a ``from_crawler()`` + method Scrapy will not call the ``from_settings()`` method so the latter + can be removed. + (:issue:`6540`) + +- The initialization API of :class:`scrapy.pipelines.media.MediaPipeline` and + its subclasses was improved and some old usage scenarios are now deprecated + (see also the "Backward-incompatible changes" section). Specifically: + + - It's deprecated to define an ``__init__()`` method that doesn't take a + ``crawler`` argument. + - It's deprecated to call an ``__init__()`` method without passing a + ``crawler`` argument. If it's passed, it's also deprecated to pass a + ``settings`` argument, which will be ignored anyway. + - Calling ``from_settings()`` is deprecated, use ``from_crawler()`` + instead. + - Overriding ``from_settings()`` is deprecated, override ``from_crawler()`` + instead. + + (:issue:`6540`) + +- The ``REQUEST_FINGERPRINTER_IMPLEMENTATION`` setting is now deprecated. + (:issue:`6212`, :issue:`6213`) + +- The ``scrapy.utils.misc.create_instance()`` function is now deprecated, use + :func:`scrapy.utils.misc.build_from_crawler` instead. + (:issue:`5523`, :issue:`5884`, :issue:`6162`, :issue:`6169`, :issue:`6540`) + +- ``scrapy.core.downloader.Downloader._get_slot_key()`` is deprecated, use :meth:`scrapy.core.downloader.Downloader.get_slot_key` instead. - (:issue:`6340`) + (:issue:`6340`, :issue:`6352`) + +- ``scrapy.utils.defer.process_chain_both()`` is now deprecated. + (:issue:`6397`) + +- ``scrapy.twisted_version`` is now deprecated, you should instead use + :attr:`twisted.version` directly (but note that it's an + ``incremental.Version`` object, not a tuple). + (:issue:`6509`, :issue:`6512`) + +- ``scrapy.utils.python.flatten()`` and ``scrapy.utils.python.iflatten()`` + are now deprecated. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.python.equal_attributes()`` is now deprecated. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.request.request_authenticate()`` is now deprecated, you + should instead just set the ``Authorization`` header directly. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.serialize.ScrapyJSONDecoder`` is now deprecated, it didn't + contain any code since Scrapy 1.0.0. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.utils.test.assert_samelines()`` is now deprecated. + (:issue:`6517`, :issue:`6519`) + +- ``scrapy.extensions.feedexport.build_storage()`` is now deprecated. You can + instead call the builder callable directly. + (:issue:`6540`) + +New features +~~~~~~~~~~~~ + +- :meth:`~scrapy.Spider.start_requests` can now yield items. + (:issue:`5289`, :issue:`6417`) + +- Added a new :class:`~scrapy.http.Response` subclass, + :class:`~scrapy.http.JsonResponse`, for responses with a `JSON MIME type + `_. + (:issue:`6069`, :issue:`6171`, :issue:`6174`) + +- The :class:`~scrapy.extensions.logstats.LogStats` extension now adds + ``items_per_minute`` and ``responses_per_minute`` to the :ref:`stats + ` when the spider closes. + (:issue:`4110`, :issue:`4111`) + +- Added :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM` which allows closing the + spider if no items were scraped in a set amount of time. + (:issue:`6434`) + +- User-defined cookies can now include the ``secure`` field. + (:issue:`6357`) + +- Added component getters to :class:`~scrapy.crawler.Crawler`: + :meth:`~scrapy.crawler.Crawler.get_addon`, + :meth:`~scrapy.crawler.Crawler.get_downloader_middleware`, + :meth:`~scrapy.crawler.Crawler.get_extension`, + :meth:`~scrapy.crawler.Crawler.get_item_pipeline`, + :meth:`~scrapy.crawler.Crawler.get_spider_middleware`. + (:issue:`6181`) + +- Slot delay updates by the :ref:`AutoThrottle extension + ` based on response latencies can now be disabled for + specific requests via the :reqmeta:`autothrottle_dont_adjust_delay` meta + key. + (:issue:`6246`, :issue:`6527`) + +- If :setting:`SPIDER_LOADER_WARN_ONLY` is set to ``True``, + :class:`~scrapy.spiderloader.SpiderLoader` does not raise + :exc:`SyntaxError` but emits a warning instead. + (:issue:`6483`, :issue:`6484`) + +- Added support for multiple-compressed responses (ones with several + encodings in the ``Content-Encoding`` header). + (:issue:`5143`, :issue:`5964`, :issue:`6063`) + +- Added support for multiple standard values in :setting:`REFERRER_POLICY`. + (:issue:`6381`) + +- Added support for brotlicffi_ (previously named brotlipy_). brotli_ is + still recommended but only brotlicffi_ works on PyPy. + (:issue:`6263`, :issue:`6269`) + + .. _brotlicffi: https://github.com/python-hyper/brotlicffi + +- Added :class:`~scrapy.contracts.default.MetadataContract` that sets the + request meta. + (:issue:`6468`, :issue:`6469`) + +Improvements +~~~~~~~~~~~~ + +- Extended the list of file extensions that + :class:`LinkExtractor ` + ignores by default. + (:issue:`6074`, :issue:`6125`) + +- :func:`scrapy.utils.httpobj.urlparse_cached` is now used in more places + instead of :func:`urllib.parse.urlparse`. + (:issue:`6228`, :issue:`6229`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.pipelines.media.MediaPipeline` is now an abstract class and + its methods that were expected to be overridden in subclasses are now + abstract methods. + (:issue:`6365`, :issue:`6368`) + +- Fixed handling of invalid ``@``-prefixed lines in contract extraction. + (:issue:`6383`, :issue:`6388`) + +- Importing ``scrapy.extensions.telnet`` no longer installs the default + reactor. + (:issue:`6432``) + +- Reduced log verbosity for dropped requests that was increased in 2.11.2. + (:issue:`6433`, :issue:`6475`) + +Documentation +~~~~~~~~~~~~~ + +- Added ``SECURITY.md`` that documents the security policy. + (:issue:`5364`, :issue:`6051`) + +- Example code for :ref:`running Scrapy from a script ` no + longer imports ``twisted.internet.reactor`` at the top level, which caused + problems with non-default reactors when this code was used unmodified. + (:issue:`6361`, :issue:`6374`) + +- Documented the :class:`~scrapy.extensions.spiderstate.SpiderState` + extension. + (:issue:`6278`, :issue:`6522`) + +- Other documentation improvements and fixes. + (:issue:`5920`, + :issue:`6094`, + :issue:`6177`, + :issue:`6200`, + :issue:`6207`, + :issue:`6216`, + :issue:`6223`, + :issue:`6317`, + :issue:`6328`, + :issue:`6389`, + :issue:`6394`, + :issue:`6402`, + :issue:`6411`, + :issue:`6427`, + :issue:`6429`, + :issue:`6440`, + :issue:`6448`, + :issue:`6449`, + :issue:`6462`, + :issue:`6497`, + :issue:`6506`, + :issue:`6507`, + :issue:`6524`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added ``py.typed``, in line with `PEP 561 + `_. + (:issue:`6058`, :issue:`6059`) + +- Fully covered the code with type hints (except for the most complicated + parts, mostly related to ``twisted.web.http`` and other Twisted parts + without type hints). + (:issue:`5989`, + :issue:`6097`, + :issue:`6127`, + :issue:`6129`, + :issue:`6130`, + :issue:`6133`, + :issue:`6143`, + :issue:`6191`, + :issue:`6268`, + :issue:`6274`, + :issue:`6275`, + :issue:`6276`, + :issue:`6279`, + :issue:`6325`, + :issue:`6326`, + :issue:`6333`, + :issue:`6335`, + :issue:`6336`, + :issue:`6337`, + :issue:`6341`, + :issue:`6353`, + :issue:`6356`, + :issue:`6370`, + :issue:`6371`, + :issue:`6384`, + :issue:`6385`, + :issue:`6387`, + :issue:`6391`, + :issue:`6395`, + :issue:`6414`, + :issue:`6422`, + :issue:`6460`, + :issue:`6466`, + :issue:`6472`, + :issue:`6494`, + :issue:`6498`, + :issue:`6516`) + +- Improved Bandit_ checks. + (:issue:`6260`, :issue:`6264`, :issue:`6265`) + +- Added pyupgrade_ to the ``pre-commit`` configuration. + (:issue:`6392`) + + .. _pyupgrade: https://github.com/asottile/pyupgrade + +- Added ``flake8-bugbear``, ``flake8-comprehensions``, ``flake8-debugger``, + ``flake8-docstrings``, ``flake8-string-format`` and + ``flake8-type-checking`` to the ``pre-commit`` configuration. + (:issue:`6406`, :issue:`6413`) + +- CI and test improvements and fixes. + (:issue:`5285`, + :issue:`5454`, + :issue:`5997`, + :issue:`6078`, + :issue:`6084`, + :issue:`6087`, + :issue:`6132`, + :issue:`6153`, + :issue:`6154`, + :issue:`6201`, + :issue:`6231`, + :issue:`6232`, + :issue:`6235`, + :issue:`6236`, + :issue:`6242`, + :issue:`6245`, + :issue:`6253`, + :issue:`6258`, + :issue:`6259`, + :issue:`6270`, + :issue:`6272`, + :issue:`6286`, + :issue:`6290`, + :issue:`6296` + :issue:`6367`, + :issue:`6372`, + :issue:`6403`, + :issue:`6416`, + :issue:`6435`, + :issue:`6489`, + :issue:`6501`, + :issue:`6504`, + :issue:`6511`, + :issue:`6543`, + :issue:`6545`) + +- Code cleanups. + (:issue:`6196`, + :issue:`6197`, + :issue:`6198`, + :issue:`6199`, + :issue:`6254`, + :issue:`6257`, + :issue:`6285`, + :issue:`6305`, + :issue:`6343`, + :issue:`6349`, + :issue:`6386`, + :issue:`6415`, + :issue:`6463`, + :issue:`6470`, + :issue:`6499`, + :issue:`6505`, + :issue:`6510`, + :issue:`6531`, + :issue:`6542`) + +Other +~~~~~ + +- Issue tracker improvements. (:issue:`6066`) .. _release-2.11.2: diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index d2fc41003..14b4aa8ba 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -157,6 +157,7 @@ Use a fallback component: .. code-block:: python from scrapy.core.downloader.handlers.http import HTTPDownloadHandler + from scrapy.utils.misc import build_from_crawler FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" @@ -167,11 +168,7 @@ Use a fallback component: def __init__(self, settings, crawler): dhcls = load_object(settings.get(FALLBACK_SETTING)) - self._fallback_handler = create_instance( - dhcls, - settings=None, - crawler=crawler, - ) + self._fallback_handler = build_from_crawler(dhcls, crawler) def download_request(self, request, spider): if request.meta.get("my_params"): diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 175c877de..f7cffb61b 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -26,7 +26,9 @@ contains a dictionary of all available extensions and their order similar to how you :ref:`configure the downloader middlewares `. -.. class:: Crawler(spidercls, settings) +.. autoclass:: Crawler + :members: get_addon, get_downloader_middleware, get_extension, + get_item_pipeline, get_spider_middleware The Crawler object must be instantiated with a :class:`scrapy.Spider` subclass and a diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 478dd9647..d34b3884b 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -4,8 +4,8 @@ Components ========== -A Scrapy component is any class whose objects are created using -:func:`scrapy.utils.misc.create_instance`. +A Scrapy component is any class whose objects are built using +:func:`~scrapy.utils.misc.build_from_crawler`. That includes the classes that you may assign to the following settings: @@ -84,3 +84,15 @@ If your requirement is a minimum Scrapy version, you may use f"method of spider middlewares as an asynchronous " f"generator." ) + +API reference +============= + +The following function can be used to create an instance of a component class: + +.. autofunction:: scrapy.utils.misc.build_from_crawler + +The following function can also be useful when implementing a component, to +report the import path of the component class, e.g. when reporting problems: + +.. autofunction:: scrapy.utils.python.global_object_name diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 9eace3be0..11a3fcb94 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -926,10 +926,6 @@ Meta tags within these tags are ignored. The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from ``[]`` to ``["noscript"]``. -.. versionchanged:: VERSION - The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from - ``[]`` to ``['noscript']``. - .. setting:: METAREFRESH_MAXDELAY METAREFRESH_MAXDELAY diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8f39bcd53..2b59cabe1 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -358,7 +358,7 @@ Acceptable values for REFERRER_POLICY - either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` subclass — a custom policy or one of the built-in ones (see classes below), -- or one of the standard W3C-defined string values, +- or one or more comma-separated standard W3C-defined string values, - or the special ``"scrapy-default"``. ======================================= ======================================================================== diff --git a/scrapy/crawler.py b/scrapy/crawler.py index de0cf543e..1ad837a47 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -184,9 +184,23 @@ class Crawler: return None def get_addon(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of an :ref:`add-on ` of + the specified class or a subclass, or ``None`` if none is found. + + .. versionadded:: 2.12 + """ return self._get_component(cls, self.addons.addons) def get_downloader_middleware(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of a :ref:`downloader middleware + ` of the specified class or a subclass, + or ``None`` if none is found. + + .. versionadded:: 2.12 + + This method can only be called after the crawl engine has been created, + e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. + """ if not self.engine: raise RuntimeError( "Crawler.get_downloader_middleware() can only be called after " @@ -195,6 +209,16 @@ class Crawler: return self._get_component(cls, self.engine.downloader.middleware.middlewares) def get_extension(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of an :ref:`extension + ` of the specified class or a subclass, + or ``None`` if none is found. + + .. versionadded:: 2.12 + + This method can only be called after the extension manager has been + created, e.g. at signals :signal:`engine_started` or + :signal:`spider_opened`. + """ if not self.extensions: raise RuntimeError( "Crawler.get_extension() can only be called after the " @@ -203,6 +227,15 @@ class Crawler: return self._get_component(cls, self.extensions.middlewares) def get_item_pipeline(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of a :ref:`item pipeline + ` of the specified class or a subclass, or + ``None`` if none is found. + + .. versionadded:: 2.12 + + This method can only be called after the crawl engine has been created, + e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. + """ if not self.engine: raise RuntimeError( "Crawler.get_item_pipeline() can only be called after the " @@ -211,6 +244,15 @@ class Crawler: return self._get_component(cls, self.engine.scraper.itemproc.middlewares) def get_spider_middleware(self, cls: type[_T]) -> _T | None: + """Return the run-time instance of a :ref:`spider middleware + ` of the specified class or a subclass, or + ``None`` if none is found. + + .. versionadded:: 2.12 + + This method can only be called after the crawl engine has been created, + e.g. at signals :signal:`engine_started` or :signal:`spider_opened`. + """ if not self.engine: raise RuntimeError( "Crawler.get_spider_middleware() can only be called after the " diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 6a7704687..0cf44aed8 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -692,8 +692,10 @@ class FeedExporter: def _get_storage( self, uri: str, feed_options: dict[str, Any] ) -> FeedStorageProtocol: - feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) - return build_from_crawler(feedcls, self.crawler, uri, feed_options=feed_options) + """Build a storage object for the specified *uri* with the specified + *feed_options*.""" + cls = self.storages.get(urlparse(uri).scheme, self.storages["file"]) + return build_from_crawler(cls, self.crawler, uri, feed_options=feed_options) def _get_uri_params( self, diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py index 6d7808c31..691a1cbf2 100644 --- a/scrapy/pipelines/media.py +++ b/scrapy/pipelines/media.py @@ -149,6 +149,15 @@ class MediaPipeline(ABC): pipe: Self if hasattr(cls, "from_settings"): pipe = cls.from_settings(crawler.settings) # type: ignore[attr-defined] + warnings.warn( + f"{global_object_name(cls)} has from_settings() and either doesn't have" + " from_crawler() or calls MediaPipeline.from_crawler() from it," + " so from_settings() was used to create the instance of it." + " This is deprecated and calling from_settings() will be removed" + " in a future Scrapy version. Please move the initialization code into" + " from_crawler() or __init__().", + category=ScrapyDeprecationWarning, + ) elif "crawler" in get_func_args(cls.__init__): pipe = cls(crawler=crawler) else: @@ -249,7 +258,7 @@ class MediaPipeline(ABC): # minimize cached information for failure result.cleanFailure() result.frames = [] - if twisted_version <= Version("twisted", 24, 10, 0): + if twisted_version < Version("twisted", 24, 10, 0): result.stack = [] # type: ignore[method-assign] # This code fixes a memory leak by avoiding to keep references to # the Request and Response objects on the Media Pipeline cache. @@ -269,9 +278,6 @@ class MediaPipeline(ABC): # To avoid keeping references to the Response and therefore Request # objects on the Media Pipeline cache, we should wipe the context of # the encapsulated exception when it is a StopIteration instance - # - # This problem does not occur in Python 2.7 since we don't have - # Exception Chaining (https://www.python.org/dev/peps/pep-3134/). context = getattr(result.value, "__context__", None) if isinstance(context, StopIteration): result.value.__context__ = None diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index a408a205d..eefadd07d 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -177,6 +177,8 @@ def build_from_crawler( ) -> T: """Construct a class instance using its ``from_crawler`` or ``from_settings`` constructor. + .. versionadded:: 2.12 + ``*args`` and ``**kwargs`` are forwarded to the constructor. Raises ``TypeError`` if the resulting instance is ``None``. diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 3864d054f..b9babb08f 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -328,8 +328,7 @@ def without_none_values( def global_object_name(obj: Any) -> str: - """ - Return full name of a global object. + """Return the full import path of the given class. >>> from scrapy import Request >>> global_object_name(Request) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 58a2d3678..cb1e2f9a1 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -454,9 +454,34 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 1) + self.assertEqual(len(w), 2) assert pipe._from_settings_called + def test_has_from_settings_and_from_crawler(self): + class Pipeline(UserDefinedPipeline): + _from_settings_called = False + _from_crawler_called = False + + @classmethod + def from_settings(cls, settings): + o = cls() + o._from_settings_called = True + return o + + @classmethod + def from_crawler(cls, crawler): + o = super().from_crawler(crawler) + o._from_crawler_called = True + return o + + with warnings.catch_warnings(record=True) as w: + pipe = Pipeline.from_crawler(self.crawler) + assert pipe.crawler == self.crawler + assert pipe._fingerprinter + self.assertEqual(len(w), 2) + assert pipe._from_settings_called + assert pipe._from_crawler_called + def test_has_from_settings_and_init(self): class Pipeline(UserDefinedPipeline): _from_settings_called = False @@ -476,7 +501,7 @@ class BuildFromCrawlerTestCase(unittest.TestCase): pipe = Pipeline.from_crawler(self.crawler) assert pipe.crawler == self.crawler assert pipe._fingerprinter - self.assertEqual(len(w), 1) + self.assertEqual(len(w), 2) assert pipe._from_settings_called assert pipe._init_called