From b1f85b5a173f48bad2881465efbab5c22aa0327d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 24 Apr 2025 20:03:36 +0500 Subject: [PATCH 1/5] Release notes for 2.13.0, up to b4c253102139e842859a9abf1455e62504cc9511. --- docs/news.rst | 198 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 193 insertions(+), 5 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 9f476ee21..b9b5ce320 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -5,21 +5,209 @@ Release notes .. _release-VERSION: -Scrapy VERSION (unreleased) ---------------------------- +Scrapy 2.13.0 (unreleased) +-------------------------- + +Highlights: + +- Added the :reqmeta:`allow_offsite` request meta key + +- HTTP/1.0 support is deprecated + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for PyPy 3.9. + (:issue:`6613`) Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - The ``from_settings()`` method of - :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware` is removed - without a deprecation period (this was needed because after the - introduction of the + :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, + deprecated in 2.12.0, is removed earlier than the usual deprecation period + (this was needed because after the introduction of the :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and switching built-in spider middlewares to it those middlewares need the :class:`~scrapy.crawler.Crawler` instance at run time). Please use ``from_crawler()`` instead. +Deprecations +~~~~~~~~~~~~ + +- Functions that were imported from :mod:`w3lib.url` and re-exported in + :mod:`scrapy.utils.url` are now deprecated, you should import them from + ``w3lib.url`` directly. They are: + + - ``scrapy.utils.url.add_or_replace_parameter()`` + + - ``scrapy.utils.url.add_or_replace_parameters()`` + + - ``scrapy.utils.url.any_to_uri()`` + + - ``scrapy.utils.url.canonicalize_url()`` + + - ``scrapy.utils.url.file_uri_to_path()`` + + - ``scrapy.utils.url.is_url()`` + + - ``scrapy.utils.url.parse_data_uri()`` + + - ``scrapy.utils.url.parse_url()`` + + - ``scrapy.utils.url.path_to_file_uri()`` + + - ``scrapy.utils.url.safe_download_url()`` + + - ``scrapy.utils.url.safe_url_string()`` + + - ``scrapy.utils.url.url_query_cleaner()`` + + - ``scrapy.utils.url.url_query_parameter()`` + + - ``scrapy.utils.url._unquotepath()`` + + - ``scrapy.utils.url._safe_chars`` attribute + + (:issue:`4577`, :issue:`6583`, :issue:`6586`) + +- HTTP/1.0 support code is deprecated. It was disabled by default and + couldn't be used together with HTTP/1.1. If you still need it, you should + write your own download handler or copy the code from Scrapy. The + deprecations include: + + - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler`` + + - ``scrapy.core.downloader.webclient.ScrapyHTTPClientFactory`` + + - ``scrapy.core.downloader.webclient.ScrapyHTTPPageGetter`` + + - Overriding + ``scrapy.core.downloader.contextfactory.ScrapyClientContextFactory.getContext()`` + + (:issue:`6634`) + +- ``scrapy.utils.versions.scrapy_components_versions()`` is deprecated, use + :func:`scrapy.utils.versions.get_versions()` instead. + (:issue:`6582`) + +- ``BaseDupeFilter.log()`` is deprecated. It does nothing and shouldn't be + called. + (:issue:`4151`) + +New features +~~~~~~~~~~~~ + +- Added the :reqmeta:`allow_offsite` request meta key that can be used + instead of the more general :attr:`~scrapy.Request.dont_filter` request + attribute to skip processing of the request by + :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` (but not + by other code that checks :attr:`~scrapy.Request.dont_filter`). + (:issue:`3690`, :issue:`6151`, :issue:`6366`) + +- :ref:`Scrapy add-ons ` can now define a class method called + ``update_pre_crawler_settings()`` to update :ref:`pre-crawler settings + `. + (:issue:`6544`, :issue:`6568`) + +- Added the :setting:`DEFAULT_DROPITEM_LOG_LEVEL` setting and the + :attr:`scrapy.exceptions.DropItem.log_level` attribute that allow + customizing the log level of the message that is logged when an item is + dropped. + (:issue:`6603`, :issue:`6608`) + +- Added the :setting:`LOG_VERSIONS` setting that allows customizing the + list of software which versions are logged when the spider starts. + (:issue:`6582`) + +Improvements +~~~~~~~~~~~~ + +- Improved the error message when running a ``scrapy`` command that requires + a project (such as ``scrapy crawl``) outside of a project directory. + (:issue:`2349`, :issue:`3426`) + +- An empty :setting:`ADDONS` setting added to the ``settings.py`` template + for new projects. + (:issue:`6587`) + +Bug fixes +~~~~~~~~~ + +- Fixed calculation of ``items_per_minute`` and ``responses_per_minute`` + stats. + (:issue:`6599`) + +- Fixed an error initializing + :class:`scrapy.extensions.feedexport.GCSFeedStorage`. + (:issue:`6617`, :issue:`6628`) + +- Fixed an error running ``scrapy bench``. + (:issue:`6632`, :issue:`6633`) + +Documentation +~~~~~~~~~~~~~ + +- Improved the contribution docs. + (:issue:`6561`, :issue:`6575`) + +- Other documentation improvements and fixes. + (:issue:`4151`, + :issue:`6526`, + :issue:`6620`, + :issue:`6621`, + :issue:`6622`, + :issue:`6623`, + :issue:`6624`) + +Packaging +~~~~~~~~~ + +- Switched from ``setup.py`` to ``pyproject.toml``. + (:issue:`6514`, :issue:`6547`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Replaced most linters with ``ruff``. + (:issue:`6565`, :issue:`6576`, :issue:`6577`, :issue:`6581`, :issue:`6584`, + :issue:`6595`, :issue:`6601`, :issue:`6631`) + +- Improved accuracy and performance of collecting test coverage. + (:issue:`6567`) + +- Fixed an error that prevented running tests from directories other than the + top level source directory. + (:issue:`6567`) + +- Reduced the amount of ``mockserver`` calls in tests to improve the overall + test run time. + (:issue:`6637`, :issue:`6648`) + +- Fixed tests that were running the same test code more than once. + (:issue:`6646`) + +- Type hints improvements and fixes. + (:issue:`6578`, :issue:`6579`, :issue:`6593`, :issue:`6605`) + +- CI and test improvements and fixes. + (:issue:`5360`, + :issue:`6271`, + :issue:`6547`, + :issue:`6560`, + :issue:`6602`, + :issue:`6607`, + :issue:`6609`, + :issue:`6613`, + :issue:`6619`, + :issue:`6626`) + +- Code cleanups. + (:issue:`6600`, + :issue:`6606`, + :issue:`6635`) + + .. _release-2.12.0: Scrapy 2.12.0 (2024-11-18) From 095140f134745960751c8c4b34da7de6b91e4a82 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 25 Apr 2025 19:43:29 +0500 Subject: [PATCH 2/5] Cover the current master in the release notes (up to daf9db7). --- docs/news.rst | 223 +++++++++++++++++++++++++++---- docs/topics/addons.rst | 2 + docs/topics/coroutines.rst | 6 + docs/topics/settings.rst | 2 +- scrapy/spidermiddlewares/base.py | 2 + 5 files changed, 210 insertions(+), 25 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index b9b5ce320..74a26c383 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,16 +3,22 @@ Release notes ============= -.. _release-VERSION: +.. _release-2.13.0: Scrapy 2.13.0 (unreleased) -------------------------- Highlights: +- The asyncio reactor is now enabled by default + - Added the :reqmeta:`allow_offsite` request meta key -- HTTP/1.0 support is deprecated +- :ref:`Spider middlewares that don't support asynchronous spider output + ` are deprecated + +- Added a base class for :ref:`universal spider middlewares + ` Modified requirements ~~~~~~~~~~~~~~~~~~~~~ @@ -23,21 +29,62 @@ Modified requirements Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +- The default value of the :setting:`TWISTED_REACTOR` setting was changed + from ``None`` to + ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. This value + was used in newly generated projects since Scrapy 2.7.0 but now existing + projects that don't explicitly set this setting will also use the asyncio + reactor. You can :ref:`change this setting in your project + ` to use a different reactor. + (:issue:`6659`, :issue:`6713`) + - The ``from_settings()`` method of :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, - deprecated in 2.12.0, is removed earlier than the usual deprecation period - (this was needed because after the introduction of the + deprecated in Scrapy 2.12.0, is removed earlier than the usual deprecation + period (this was needed because after the introduction of the :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and switching built-in spider middlewares to it those middlewares need the :class:`~scrapy.crawler.Crawler` instance at run time). Please use ``from_crawler()`` instead. + (:issue:`6693`) + +- ``scrapy.utils.url.escape_ajax()`` is no longer called when a + :class:`~scrapy.Request` instance is created. It was only useful for + websites supporting the ``_escaped_fragment_`` feature which most modern + websites don't support. If you still need this you can modify the URLs + before passing them to :class:`~scrapy.Request`. + (:issue:`6523`, :issue:`6651`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed old deprecated name aliases for some signals: + + - ``stats_spider_opened`` (use ``spider_opened`` instead) + + - ``stats_spider_closing`` and ``stats_spider_closed`` (use + ``spider_closed`` instead) + + - ``item_passed`` (use ``item_scraped`` instead) + + - ``request_received`` (use ``request_scheduled`` instead) + + (:issue:`6654`, :issue:`6655`) Deprecations ~~~~~~~~~~~~ +- :ref:`Spider middlewares that don't support asynchronous spider output + ` are deprecated. The async iterable + downgrading feature, needed for using such middlewares with asynchronous + callbacks and with other spider middlewares that produce asynchronous + iterables, is also deprecated. Please update all such middlewares to + support asynchronous spider output. + (:issue:`6664`) + - Functions that were imported from :mod:`w3lib.url` and re-exported in :mod:`scrapy.utils.url` are now deprecated, you should import them from - ``w3lib.url`` directly. They are: + :mod:`w3lib.url` directly. They are: - ``scrapy.utils.url.add_or_replace_parameter()`` @@ -65,10 +112,6 @@ Deprecations - ``scrapy.utils.url.url_query_parameter()`` - - ``scrapy.utils.url._unquotepath()`` - - - ``scrapy.utils.url._safe_chars`` attribute - (:issue:`4577`, :issue:`6583`, :issue:`6586`) - HTTP/1.0 support code is deprecated. It was disabled by default and @@ -87,6 +130,37 @@ Deprecations (:issue:`6634`) +- The following modules and functions used only in tests are deprecated: + + - the ``scrapy/utils/testproc`` module + + - the ``scrapy/utils/testsite`` module + + - ``scrapy.utils.test.assert_gcs_environ()`` + + - ``scrapy.utils.test.get_ftp_content_and_delete()`` + + - ``scrapy.utils.test.get_gcs_content_and_delete()`` + + - ``scrapy.utils.test.mock_google_cloud_storage()`` + + - ``scrapy.utils.test.skip_if_no_boto()`` + + If you need to use them in your tests or code, you can copy the code from Scrapy. + (:issue:`6696`) + +- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` is + deprecated. It was disabled by default and isn't useful for most of the + existing websites. + (:issue:`6523`, :issue:`6651`, :issue:`6656`) + +- ``scrapy.utils.url.escape_ajax()`` is deprecated. + (:issue:`6523`, :issue:`6651`) + +- ``scrapy.spiders.init.InitSpider`` is deprecated. If you find it useful, + you can copy its code from Scrapy. + (:issue:`6708`, :issue:`6714`) + - ``scrapy.utils.versions.scrapy_components_versions()`` is deprecated, use :func:`scrapy.utils.versions.get_versions()` instead. (:issue:`6582`) @@ -105,29 +179,63 @@ New features by other code that checks :attr:`~scrapy.Request.dont_filter`). (:issue:`3690`, :issue:`6151`, :issue:`6366`) +- Added an optional base class for spider middlewares, + :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which can be + helpful for writing :ref:`universal spider middlewares + ` without boilerplate and code duplication. + The built-in spider middlewares now inherit from this class. + (:issue:`6693`) + - :ref:`Scrapy add-ons ` can now define a class method called ``update_pre_crawler_settings()`` to update :ref:`pre-crawler settings `. (:issue:`6544`, :issue:`6568`) +- Added :ref:`helpers ` for modifying :ref:`component + priority dictionary ` settings. + (:issue:`6614`) + +- Responses that use an unknown/unsupported encoding now produce a warning. + If Scrapy knows that installing an additional package (such as brotli_) + will allow decoding the response, that will be mentioned in the warning. + (:issue:`4697`, :issue:`6618`) + +- Added the ``spider_exceptions/count`` stat which tracks the total count of + exceptions (tracked also by per-type ``spider_exceptions/*`` stats). + (:issue:`6739`, :issue:`6740`) + - Added the :setting:`DEFAULT_DROPITEM_LOG_LEVEL` setting and the :attr:`scrapy.exceptions.DropItem.log_level` attribute that allow customizing the log level of the message that is logged when an item is dropped. (:issue:`6603`, :issue:`6608`) +- Added support for the ``-b, --cookie`` curl argument to + :meth:`scrapy.Request.from_curl`. + (:issue:`6684`) + - Added the :setting:`LOG_VERSIONS` setting that allows customizing the - list of software which versions are logged when the spider starts. + list of software whose versions are logged when the spider starts. (:issue:`6582`) +- Added the :setting:`WARN_ON_GENERATOR_RETURN_VALUE` setting that allows + disabling run time analysis of callback code used to warn about incorrect + ``return`` statements in generator-based callbacks. You may need to disable + this setting if this analysis breaks on your callback code. + (:issue:`6731`, :issue:`6738`) + Improvements ~~~~~~~~~~~~ +- Removed or postponed some calls of :func:`itemadapter.is_item` to increase + performance. + (:issue:`6719`) + - Improved the error message when running a ``scrapy`` command that requires a project (such as ``scrapy crawl``) outside of a project directory. (:issue:`2349`, :issue:`3426`) -- An empty :setting:`ADDONS` setting added to the ``settings.py`` template +- Added an empty :setting:`ADDONS` setting to the ``settings.py`` template for new projects. (:issue:`6587`) @@ -145,12 +253,46 @@ Bug fixes - Fixed an error running ``scrapy bench``. (:issue:`6632`, :issue:`6633`) +- Fixed duplicated log messages about the reactor and the event loop. + (:issue:`6636`, :issue:`6657`) + +- Fixed resolving type annotations of ``SitemapSpider._parse_sitemap()`` at + run time, required by tools such as scrapy-poet_. + (:issue:`6665`, :issue:`6671`) + + .. _scrapy-poet: https://github.com/scrapinghub/scrapy-poet + +- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without + an installed reactor now raises an exception instead of installing a + reactor. + (:issue:`6732`, :issue:`6735`) + +- Restored support for the ``x-gzip`` content encoding. + (:issue:`6618`) + Documentation ~~~~~~~~~~~~~ -- Improved the contribution docs. +- Improved the :ref:`docs ` about asynchronous + iterable support in spider middlewares. + (:issue:`6688`) + +- Improved the :ref:`docs ` about using + :class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based + code. + (:issue:`6734`) + +- Improved the :ref:`contribution docs `. (:issue:`6561`, :issue:`6575`) +- Removed the ``Splash`` recommendation from the :ref:`headless browser + ` suggestion. We no longer recommend using + ``Splash`` and recommend using other headless browser solutions instead. + (:issue:`6642`, :issue:`6701`) + +- Added the dark mode to the HTML documentation. + (:issue:`6653`) + - Other documentation improvements and fixes. (:issue:`4151`, :issue:`6526`, @@ -158,7 +300,9 @@ Documentation :issue:`6621`, :issue:`6622`, :issue:`6623`, - :issue:`6624`) + :issue:`6624`, + :issue:`6721`, + :issue:`6723`) Packaging ~~~~~~~~~ @@ -169,12 +313,20 @@ Packaging Quality assurance ~~~~~~~~~~~~~~~~~ -- Replaced most linters with ``ruff``. - (:issue:`6565`, :issue:`6576`, :issue:`6577`, :issue:`6581`, :issue:`6584`, - :issue:`6595`, :issue:`6601`, :issue:`6631`) +- Replaced most linters with ruff_. + (:issue:`6565`, + :issue:`6576`, + :issue:`6577`, + :issue:`6581`, + :issue:`6584`, + :issue:`6595`, + :issue:`6601`, + :issue:`6631`) + + .. _ruff: https://docs.astral.sh/ruff/ - Improved accuracy and performance of collecting test coverage. - (:issue:`6567`) + (:issue:`6255`, :issue:`6610`) - Fixed an error that prevented running tests from directories other than the top level source directory. @@ -185,10 +337,28 @@ Quality assurance (:issue:`6637`, :issue:`6648`) - Fixed tests that were running the same test code more than once. - (:issue:`6646`) + (:issue:`6646`, :issue:`6647`, :issue:`6650`) + +- Refactored tests to use more ``pytest`` features instead of ``unittest`` + ones where possible. + (:issue:`6678`, + :issue:`6680`, + :issue:`6695`, + :issue:`6699`, + :issue:`6700`, + :issue:`6702`, + :issue:`6709`, + :issue:`6710`, + :issue:`6711`, + :issue:`6712`, + :issue:`6725`) - Type hints improvements and fixes. - (:issue:`6578`, :issue:`6579`, :issue:`6593`, :issue:`6605`) + (:issue:`6578`, + :issue:`6579`, + :issue:`6593`, + :issue:`6605`, + :issue:`6694`) - CI and test improvements and fixes. (:issue:`5360`, @@ -200,7 +370,16 @@ Quality assurance :issue:`6609`, :issue:`6613`, :issue:`6619`, - :issue:`6626`) + :issue:`6626`, + :issue:`6679`, + :issue:`6703`, + :issue:`6704`, + :issue:`6716`, + :issue:`6720`, + :issue:`6722`, + :issue:`6724`, + :issue:`6741`, + :issue:`6743`) - Code cleanups. (:issue:`6600`, @@ -806,8 +985,6 @@ Bug fixes - Restored support for brotlipy_, which had been dropped in Scrapy 2.11.1 in favor of brotli_. (:issue:`6261`) - .. _brotli: https://github.com/google/brotli - .. note:: brotlipy is deprecated, both in Scrapy and upstream. Use brotli instead if you can. @@ -2391,8 +2568,6 @@ Scrapy 2.5.1 (2021-10-05) need to upgrade scrapy-splash to a greater version for it to continue to work. -.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash - .. _release-2.5.0: diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 17e3c177a..815501e66 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -124,6 +124,8 @@ Set some basic configuration: "ITEM_PIPELINES", MyPipeline, 200 ) +.. _priority-dict-helpers: + .. tip:: When editing a :ref:`component priority dictionary ` setting, like :setting:`ITEM_PIPELINES`, consider using setting methods like diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 1c80857f6..1a84f893c 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -344,3 +344,9 @@ For example: feature will be removed, and all spider middlewares will be expected to define their ``process_spider_output`` method as an asynchronous generator. + +Since 2.13.0, Scrapy provides a base class, +:class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which implements +the ``process_spider_output()`` and ``process_spider_output_async()`` methods, +so instead of duplicating the processing code you can override the +``get_processed_request()`` and/or the ``get_processed_item()`` method. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index a59a61050..73ac36646 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -2005,7 +2005,7 @@ current platform. ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated ``settings.py`` file. -.. versionchanged:: VERSION +.. versionchanged:: 2.13 The default value was changed from ``None`` to ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index 650192095..5e4370d45 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -17,6 +17,8 @@ if TYPE_CHECKING: class BaseSpiderMiddleware: """Optional base class for spider middlewares. + .. versionadded:: 2.13 + This class provides helper methods for asynchronous ``process_spider_output`` methods. Middlewares that don't have a ``process_spider_output`` method don't need to use it. From 4aba7e5f6675703159220ee22bba6953c5685ef6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 25 Apr 2025 20:16:26 +0500 Subject: [PATCH 3/5] Mention the deprecation of TestSpider. --- docs/news.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/news.rst b/docs/news.rst index 74a26c383..7bb25e6b6 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -149,6 +149,11 @@ Deprecations If you need to use them in your tests or code, you can copy the code from Scrapy. (:issue:`6696`) +- ``scrapy.utils.test.TestSpider`` is deprecated. If you need an empty spider + class you can use :class:`scrapy.utils.spider.DefaultSpider` or create your + own subclass of :class:`scrapy.Spider`. + (:issue:`6678`) + - ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` is deprecated. It was disabled by default and isn't useful for most of the existing websites. From 5f6d1b464b81e9673a9639fb8f742d53831c98dc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 5 May 2025 18:08:54 +0500 Subject: [PATCH 4/5] Cover up to 373e501 in the release notes. --- docs/news.rst | 46 ++++++++++++++++++++++++++++++++++------ docs/topics/settings.rst | 3 ++- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 7bb25e6b6..d9bc572be 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -26,6 +26,9 @@ Modified requirements - Dropped support for PyPy 3.9. (:issue:`6613`) +- Added support for PyPy 3.11. + (:issue:`6697`) + Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -38,6 +41,10 @@ Backward-incompatible changes ` to use a different reactor. (:issue:`6659`, :issue:`6713`) +- The URL length limit, set by the :setting:`URLLENGTH_LIMIT` setting, is now + also enforced for start requests. + (:issue:`6777`) + - The ``from_settings()`` method of :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, deprecated in Scrapy 2.12.0, is removed earlier than the usual deprecation @@ -174,6 +181,19 @@ Deprecations called. (:issue:`4151`) +- Passing the ``spider`` argument to the following methods of + :class:`~scrapy.core.scraper.Scraper` is deprecated: + + - ``close_spider()`` + + - ``enqueue_scrape()`` + + - ``handle_spider_error()`` + + - ``handle_spider_output()`` + + (:issue:`6764`) + New features ~~~~~~~~~~~~ @@ -189,7 +209,7 @@ New features helpful for writing :ref:`universal spider middlewares ` without boilerplate and code duplication. The built-in spider middlewares now inherit from this class. - (:issue:`6693`) + (:issue:`6693`, :issue:`6777`) - :ref:`Scrapy add-ons ` can now define a class method called ``update_pre_crawler_settings()`` to update :ref:`pre-crawler settings @@ -278,14 +298,17 @@ Bug fixes Documentation ~~~~~~~~~~~~~ +- Documented the setting values set in the default project template. + (:issue:`6762`, :issue:`6775`) + - Improved the :ref:`docs ` about asynchronous iterable support in spider middlewares. (:issue:`6688`) - Improved the :ref:`docs ` about using :class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based - code. - (:issue:`6734`) + code and included a list of such APIs. + (:issue:`6677`, :issue:`6734`, :issue:`6776`) - Improved the :ref:`contribution docs `. (:issue:`6561`, :issue:`6575`) @@ -307,7 +330,8 @@ Documentation :issue:`6623`, :issue:`6624`, :issue:`6721`, - :issue:`6723`) + :issue:`6723`, + :issue:`6780`) Packaging ~~~~~~~~~ @@ -315,6 +339,11 @@ Packaging - Switched from ``setup.py`` to ``pyproject.toml``. (:issue:`6514`, :issue:`6547`) +- Switched the build backend from setuptools_ to hatchling_. + (:issue:`6771`) + + .. _hatchling: https://pypi.org/project/hatchling/ + Quality assurance ~~~~~~~~~~~~~~~~~ @@ -384,12 +413,17 @@ Quality assurance :issue:`6722`, :issue:`6724`, :issue:`6741`, - :issue:`6743`) + :issue:`6743`, + :issue:`6766`, + :issue:`6770`, + :issue:`6772`, + :issue:`6773`) - Code cleanups. (:issue:`6600`, :issue:`6606`, - :issue:`6635`) + :issue:`6635`, + :issue:`6764`) .. _release-2.12.0: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 73ac36646..fec82f8e3 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -701,7 +701,8 @@ connections (for ``HTTP10DownloadHandler``). .. note:: - HTTP/1.0 is rarely used nowadays so you can safely ignore this setting, + HTTP/1.0 is rarely used nowadays and its Scrapy support is deprecated, + so you can safely ignore this setting, unless you really want to use HTTP/1.0 and override :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly, i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``. From 0ce693dfa91a4ebf8418fd04c5595a570939e480 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 8 May 2025 13:23:19 +0500 Subject: [PATCH 5/5] Update VERSION strings. --- docs/topics/coroutines.rst | 4 ++-- docs/topics/spider-middleware.rst | 2 +- scrapy/core/spidermw.py | 20 ++++++++++---------- scrapy/spiders/__init__.py | 4 ++-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 62cbc3d49..00812ed7f 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -21,7 +21,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be defined as an :term:`asynchronous generator`. - .. versionadded: VERSION + .. versionadded: 2.13 - :class:`~scrapy.Request` callbacks. @@ -59,7 +59,7 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): of :ref:`spider middlewares `, which *must* be defined as an :term:`asynchronous generator`. - .. versionadded:: VERSION + .. versionadded:: 2.13 - :ref:`Signal handlers that support deferreds `. diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 638035e64..67178b8fd 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -86,7 +86,7 @@ one or more of these methods: You may yield the same type of objects as :meth:`~scrapy.Spider.start`. To write spider middlewares that work on Scrapy versions lower than - VERSION, define also a synchronous ``process_start_requests()`` method + 2.13, define also a synchronous ``process_start_requests()`` method that returns an iterable. For example: .. code-block:: python diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 4a0cd9464..310abb9b7 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -85,8 +85,8 @@ class SpiderMiddlewareManager(MiddlewareManager): "either disable or make universal 1 of those 2 sets of " "spider middlewares. Making a spider middleware universal " "means having it define both methods. See the release notes " - "of Scrapy VERSION for details: " - "https://docs.scrapy.org/en/VERSION/news.html" + "of Scrapy 2.13 for details: " + "https://docs.scrapy.org/en/2.13/news.html" ) self._use_start_requests = bool(deprecated_middlewares) @@ -103,15 +103,15 @@ class SpiderMiddlewareManager(MiddlewareManager): f"been deprecated in favor of a new method, process_start(), " f"to support asynchronous code execution. " f"process_start_requests() will stop being called in a future " - f"version of Scrapy. If you use Scrapy VERSION or higher " + f"version of Scrapy. If you use Scrapy 2.13 or higher " f"only, replace process_start_requests() with " f"process_start(); note that process_start() is a coroutine " f"(async def). If you need to maintain compatibility with " f"lower Scrapy versions, when defining " f"process_start_requests() in a spider middleware class, " f"define process_start() as well. See the release notes of " - f"Scrapy VERSION for details: " - f"https://docs.scrapy.org/en/VERSION/news.html", + f"Scrapy 2.13 for details: " + f"https://docs.scrapy.org/en/2.13/news.html", ScrapyDeprecationWarning, ) @@ -435,15 +435,15 @@ class SpiderMiddlewareManager(MiddlewareManager): f"start_requests() has been deprecated in favor of a new " f"method, start(), to support asynchronous code " f"execution. start_requests() will stop being called in a " - f"future version of Scrapy. If you use Scrapy VERSION or " + f"future version of Scrapy. If you use Scrapy 2.13 or " f"higher only, replace start_requests() with start(); " f"note that start() is a coroutine (async def). If you " f"need to maintain compatibility with lower Scrapy versions, " f"when overriding start_requests() in a spider class, " f"override start() as well; you can use super() to " f"reuse the inherited start() implementation without " - f"copy-pasting. See the release notes of Scrapy VERSION for " - f"details: https://docs.scrapy.org/en/VERSION/news.html", + f"copy-pasting. See the release notes of Scrapy 2.13 for " + f"details: https://docs.scrapy.org/en/2.13/news.html", ScrapyDeprecationWarning, ) @@ -469,8 +469,8 @@ class SpiderMiddlewareManager(MiddlewareManager): f"deprecated spider middlewares (and earlier Scrapy versions) " f"by defining a sync start_requests() method that works " f"similarly to its existing start() method. See the " - f"release notes of Scrapy VERSION for details: " - f"https://docs.scrapy.org/en/VERSION/news.html" + f"release notes of Scrapy 2.13 for details: " + f"https://docs.scrapy.org/en/2.13/news.html" ) # This method is only needed until _async compatibility methods are removed. diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 0a1d85ae6..a722dd83b 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -83,7 +83,7 @@ class Spider(object_ref): async def start(self) -> AsyncIterator[Any]: """Yield the initial :class:`~scrapy.Request` objects to send. - .. versionadded:: VERSION + .. versionadded:: 2.13 For example: @@ -115,7 +115,7 @@ class Spider(object_ref): async def start(self): yield {"foo": "bar"} - To write spiders that work on Scrapy versions lower than VERSION, + To write spiders that work on Scrapy versions lower than 2.13, define also a synchronous ``start_requests()`` method that returns an iterable. For example: