From 5ccc8dbe8a735b322bacca16848ab9dfed839b99 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 7 Jul 2026 14:57:42 +0500 Subject: [PATCH] Release notes for 2.17.0 (#7723) * Cover 2.17.0 in the release notes up to dd10cb8. * Fix a copypasted issue number. * Address minor issues * Minor fixes * Cover additional deprecations introduced while improving test coverage * versionadded/versionchanged improvements * Add a versionadded to the give_up_log_level docstring --------- Co-authored-by: Adrian Chaves --- docs/news.rst | 308 +++++++++++++++++++++++++- docs/topics/autothrottle.rst | 2 + docs/topics/download-handlers.rst | 4 +- docs/topics/downloader-middleware.rst | 8 + docs/topics/feed-exports.rst | 4 - docs/topics/request-response.rst | 11 +- docs/topics/settings.rst | 7 + scrapy/downloadermiddlewares/retry.py | 3 + scrapy/item.py | 2 +- scrapy/linkextractors/lxmlhtml.py | 4 +- 10 files changed, 341 insertions(+), 12 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index b8b976df9..6966e7763 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,310 @@ Release notes ============= +.. _release-2.17.0: + +Scrapy 2.17.0 (unreleased) +-------------------------- + +Highlights: + +- HTTP/2 and SOCKS proxy support for ``HttpxDownloadHandler`` + +- Improved settings for changing allowed TLS versions + +Deprecations +~~~~~~~~~~~~ + +- The ``DOWNLOADER_CLIENT_TLS_METHOD`` setting is deprecated. You should use + the :setting:`DOWNLOAD_TLS_MIN_VERSION` and/or + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings instead if you want to change + the TLS method selection. + (:issue:`3288`, :issue:`6546`) + +- The following spider attributes are deprecated in favor of settings: + + - ``http_user`` (use :setting:`HTTPAUTH_USER`) + + - ``http_pass`` (use :setting:`HTTPAUTH_PASS`) + + - ``http_auth_domain`` (use :setting:`HTTPAUTH_DOMAIN`) + + (:issue:`7590`) + +- The ``scrapy.commands.ScrapyCommand.help()`` method is deprecated. It was + never called by Scrapy. + (:issue:`7626`, :issue:`7633`) + +- The following TLS-related functions and constants, intended for internal + use, are deprecated: + + - ``scrapy.core.downloader.tls.METHOD_TLS`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv10`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv11`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv12`` + + - ``scrapy.core.downloader.tls.openssl_methods`` + + - ``scrapy.core.downloader.tls.DEFAULT_CIPHERS`` + + - ``scrapy.utils.ssl.ffi_buf_to_string()`` + + - ``scrapy.utils.ssl.get_temp_key_info()`` + + - ``scrapy.utils.ssl.x509name_to_string()`` + + (:issue:`6546`, :issue:`7619`, :issue:`7665`) + +- The ``CRAWLSPIDER_FOLLOW_LINKS`` setting is deprecated. You can set + ``follow=False`` in your rules to achieve the same effect. + (:issue:`7592`) + +- Instantiating + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + without a ``crawler`` argument is deprecated. + (:issue:`7655`) + +- Instantiating + :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` without a + ``settings`` argument is deprecated. + (:issue:`7664`) + +New features +~~~~~~~~~~~~ + +- Added support for HTTP/2 requests to + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. It + requires setting the new :setting:`HTTPX_HTTP2_ENABLED` setting to + ``True``. + (:issue:`7575`) + +- Added support for SOCKS proxies to + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. + (:issue:`747`, :issue:`7575`) + +- Added :setting:`DOWNLOAD_TLS_MIN_VERSION` and + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings as replacements for the + ``DOWNLOADER_CLIENT_TLS_METHOD`` setting (which is now deprecated). + Compared to the old setting, they support specifying a range of allowed + versions and support newer TLS versions. + (:issue:`4821`, :issue:`6546`) + +- Added :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS` and + :setting:`HTTPAUTH_DOMAIN` settings and :reqmeta:`http_user`, + :reqmeta:`http_pass` and :reqmeta:`http_auth_domain` meta keys as more + flexible ways to set HTTP authentication data. + (:issue:`7590`) + +- Added a :reqmeta:`verbatim_url` meta key that can be set to ``True`` to + skip request URL canonicalization. + (:issue:`7473`) + +- Added ``deny_tags`` and ``deny_attrs`` arguments to :class:`LinkExtractor + `. + (:issue:`6321`, :issue:`7679`) + +- :attr:`scrapy.Item.fields` now returns the fields in the definition order + instead of the alphabetical one. + (:issue:`7015`, :issue:`7694`) + +- Added a :setting:`RETRY_GIVE_UP_LOG_LEVEL` setting, a + :reqmeta:`give_up_log_level` meta key and a ``give_up_log_level`` argument + of the + :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function that + allow changing the log level of the message logged when the retry limit has + been reached. + (:issue:`4622`, :issue:`5297`, :issue:`7567`) + +- It's now possible to set :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` to + ``None`` to use the default ciphers of the underlying TLS implementation. + (:issue:`7499`, :issue:`7665`) + +Improvements +~~~~~~~~~~~~ + +- :class:`~scrapy.FormRequest` is no longer deprecated, only its + ``from_response()`` method is still deprecated. + (:issue:`7561`, :issue:`7671`) + +- Switched the item definition in the default project template from a + :class:`scrapy.item.Item` to a dataclass. + (:issue:`7493`, :issue:`7513`) + +- Fixed deprecation warnings with pyOpenSSL 26.3.0. + (:issue:`7619`) + +- Removed the runtime warnings for :attr:`Spider.allowed_domains + ` containing URLs or domains with ports + instead of just domains and for spider classes having a ``start_url`` + attribute instead of :class:`~scrapy.spiders.Spider.start_urls`. Please use + :doc:`scrapy-lint ` to find mistakes in your spider code + instead. + (:issue:`4421`, :issue:`7627`) + +- :func:`scrapy.utils.test.get_crawler` now disables + :setting:`TELNETCONSOLE_ENABLED` by default. + (:issue:`7644`) + +- Other code refactoring and improvements. + (:issue:`7409`, :issue:`7593`, :issue:`7594`, :issue:`7611`, :issue:`7649`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` no + longer ignores proxy credentials for redirected or retried requests. + (:issue:`7601`, :issue:`7630`) + +- :class:`~scrapy.extensions.feedexport.GCSFeedStorage` now closes the + temporary file after the upload. + (:issue:`7546`) + +- Fixed ``scrapy shell `` running a full spider crawl when there is a + spider for the requested URL. This bug was introduced in Scrapy 2.13.0. + (:issue:`7552`, :issue:`7557`) + +- The :setting:`IMAGES_STORE_S3_ACL` and :setting:`IMAGES_STORE_GCS_ACL` + settings are no longer ignored. This bug was introduced in Scrapy 2.12.0. + (:issue:`7597`, :issue:`7614`) + +- :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` now closes + the connection after making the request. + (:issue:`7602`, :issue:`7667`) + +- Removed the deprecated ``spider`` argument from the pipeline defined in the + default project template. + (:issue:`7676`) + +- Fixed ``scrapy genspider --edit`` not working. + (:issue:`7260`, :issue:`7683`) + +- When a :class:`~scrapy.crawler.Crawler` instance is passed to + :meth:`AsyncCrawlerRunner.create_crawler() + ` or + :meth:`CrawlerRunner.create_crawler() + `, settings from both classes + are now merged, previously only the settings from the + :class:`~scrapy.crawler.Crawler` instance were used. + (:issue:`1280`, :issue:`7647`) + +- Fixed several issues with cookie handling in + :func:`scrapy.utils.request.request_to_curl`. + (:issue:`7603`, :issue:`7675`, :issue:`7684`) + +- Fixed :class:`scrapy.resolver.CachingThreadedResolver` not disabling the + cache when :setting:`DNSCACHE_ENABLED` is set to ``False``. + (:issue:`7663`) + +- Fixed :func:`scrapy.utils.response.open_in_browser` not removing comments + when looking for the ```` tag. + (:issue:`7506`) + +- Fixed checking for deprecated methods in custom :setting:`ITEM_PROCESSOR` + implementations. + (:issue:`7589`) + +- Fixed :func:`scrapy.utils.url.strip_url` corrupting some URLs with + credentials. + (:issue:`7604`, :issue:`7605`) + +- :func:`scrapy.utils.misc.rel_has_nofollow` now ignores the case when + looking for "nofollow" strings. + (:issue:`7632`) + +- Fixed an exception in :class:`scrapy.utils.sitemap.Sitemap` when parsing + some malformed sitemaps. + (:issue:`7686`, :issue:`7687`) + +Documentation +~~~~~~~~~~~~~ + +- Mentioned :doc:`scrapy-lint ` in the docs. + (:issue:`4421`, :issue:`7627`) + +- Added the docs about :ref:`security considerations `. + (:issue:`7389`, :issue:`7678`) + +- Improved the :ref:`item pipeline docs `. + (:issue:`2350`, :issue:`7676`) + +- Documented which stats are collected by + :class:`~scrapy.extensions.corestats.CoreStats`. + (:issue:`7421`) + +- Switched documentation examples from using :class:`scrapy.item.Item` to + using dataclasses. + (:issue:`7493`, :issue:`7513`) + +- Added feature comparison tables to the :ref:`download handler + ` docs. + (:issue:`7575`) + +- Improved the docs for :ref:`logging settings `. + (:issue:`6909`, :issue:`7668`) + +- Documented a way to :ref:`improve startup time and memory usage + ` by using :setting:`SPIDER_MODULES`. + (:issue:`7576`, :issue:`7600`) + +- Clarified handling of the ``type`` argument of :class:`~scrapy.Selector`. + (:issue:`7704`) + +- Other documentation improvements and fixes. + (:issue:`4954`, + :issue:`6120`, + :issue:`7286`, + :issue:`7564`, + :issue:`7573`, + :issue:`7598`, + :issue:`7599`, + :issue:`7698`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Fixed deprecation warnings with pytest 9.1.0. + (:issue:`7621`) + +- Type hints improvements and fixes. + (:issue:`6958`, :issue:`7586`) + +- CI and test improvements and fixes. + (:issue:`5954`, + :issue:`7002`, + :issue:`7017`, + :issue:`7247`, + :issue:`7508`, + :issue:`7545`, + :issue:`7566`, + :issue:`7574`, + :issue:`7585`, + :issue:`7595`, + :issue:`7608`, + :issue:`7610`, + :issue:`7612`, + :issue:`7616`, + :issue:`7625`, + :issue:`7637`, + :issue:`7639`, + :issue:`7640`, + :issue:`7641`, + :issue:`7642`, + :issue:`7643`, + :issue:`7644`, + :issue:`7645`, + :issue:`7646`, + :issue:`7654`, + :issue:`7655`, + :issue:`7664`, + :issue:`7672`, + :issue:`7677`, + :issue:`7680`, + :issue:`7682`, + :issue:`7692`) + .. _release-2.16.0: Scrapy 2.16.0 (2026-05-19) @@ -6243,8 +6547,8 @@ Backward-incompatible changes ``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly. * :class:`~scrapy.crawler.Crawler`, - :class:`CrawlerRunner.crawl ` and - :class:`CrawlerRunner.create_crawler ` + :meth:`CrawlerRunner.crawl ` and + :meth:`CrawlerRunner.create_crawler ` no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they only accept a :class:`~scrapy.spiders.Spider` subclass now. diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index d0321c906..787ef842a 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -88,6 +88,8 @@ server) is, and this extension builds on that premise. Prevent specific requests from triggering slot delay adjustments ================================================================ +.. versionadded:: 2.12.0 + AutoThrottle adjusts the delay of download slots based on the latencies of responses that belong to that download slot. The only exceptions are non-200 responses, which are only taken into account to increase that delay, but diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index c901e01e4..9bbfee9e2 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -329,13 +329,13 @@ Other limitations: HTTPX_HTTP2_ENABLED ^^^^^^^^^^^^^^^^^^^ +.. versionadded:: 2.17.0 + Default: ``False`` Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra needs to be installed if you want to enable this setting. -.. versionadded:: VERSION - Built-in non-HTTP download handlers reference ============================================= diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 5649453b1..054c1130c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -351,6 +351,8 @@ HttpAuthMiddleware HTTPAUTH_USER ~~~~~~~~~~~~~ +.. versionadded:: 2.17.0 + Default: ``""`` The username to use for HTTP basic authentication, applied to all requests @@ -361,6 +363,8 @@ whose URL matches :setting:`HTTPAUTH_DOMAIN`. HTTPAUTH_PASS ~~~~~~~~~~~~~ +.. versionadded:: 2.17.0 + Default: ``""`` The password to use for HTTP basic authentication. @@ -370,6 +374,8 @@ The password to use for HTTP basic authentication. HTTPAUTH_DOMAIN ~~~~~~~~~~~~~~~ +.. versionadded:: 2.17.0 + Default: ``None`` The domain (and its subdomains) to which HTTP basic authentication credentials @@ -1104,6 +1110,8 @@ exception propagation, see RETRY_GIVE_UP_LOG_LEVEL ^^^^^^^^^^^^^^^^^^^^^^^ +.. versionadded:: 2.17.0 + Default: ``"ERROR"`` :ref:`Logging level ` used for the message logged when a request diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 6ce5255b9..9aa1b8d5f 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -528,10 +528,6 @@ safe numeric encoding (``\uXXXX`` sequences) for historic reasons. Use ``"utf-8"`` if you want UTF-8 for JSON too. -.. versionchanged:: 2.8 - The :command:`startproject` command now sets this setting to - ``"utf-8"`` in the generated ``settings.py`` file. - .. setting:: FEED_EXPORT_FIELDS FEED_EXPORT_FIELDS diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index f2fcfb0b5..44bc09b7f 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -806,6 +806,8 @@ Whether or not to fail on broken responses. See: give_up_log_level ----------------- +.. versionadded:: 2.17.0 + :ref:`Logging level ` used for the message logged when a request exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details. @@ -814,6 +816,8 @@ exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details. http_auth_domain ---------------- +.. versionadded:: 2.17.0 + Overrides :setting:`HTTPAUTH_DOMAIN` for this request. .. reqmeta:: http_pass @@ -821,6 +825,8 @@ Overrides :setting:`HTTPAUTH_DOMAIN` for this request. http_pass --------- +.. versionadded:: 2.17.0 + Overrides :setting:`HTTPAUTH_PASS` for this request. .. reqmeta:: http_user @@ -828,6 +834,8 @@ Overrides :setting:`HTTPAUTH_PASS` for this request. http_user --------- +.. versionadded:: 2.17.0 + Overrides :setting:`HTTPAUTH_USER` for this request. .. reqmeta:: max_retry_times @@ -844,6 +852,8 @@ The meta key is used set retry times per request. When set, the verbatim_url ------------ +.. versionadded:: 2.17.0 + Set this key to ``True`` to keep the request URL as passed to :class:`~scrapy.Request`, without URL percent-encoding. @@ -854,7 +864,6 @@ characters that would otherwise be canonicalized get different fingerprints. In this mode, the ``keep_fragments`` parameter is ignored, and it is effectively true. - .. _topics-stop-response-download: Stopping the download of a Response diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 11251a92c..a5148ee26 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -766,6 +766,9 @@ specific cipher that is not included in ``DEFAULT`` if a website requires it. Set this setting to ``None`` to use the default ciphers of the underlying TLS implementation. +.. versionchanged:: 2.17.0 + Added support for setting this to ``None``. + .. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format .. note:: @@ -781,6 +784,8 @@ implementation. DOWNLOAD_TLS_MAX_VERSION ------------------------ +.. versionadded:: 2.17.0 + Default: ``None`` Use this setting to change the maximum version of the TLS protocol allowed to @@ -816,6 +821,8 @@ modern environments. DOWNLOAD_TLS_MIN_VERSION ------------------------ +.. versionadded:: 2.17.0 + Default: ``None`` Use this setting to change the minimum version of the TLS protocol allowed to diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index a0a0b60a2..e1dcd90d5 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -87,6 +87,9 @@ def get_retry_request( message logged when a request exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details. + .. versionadded:: 2.17.0 + The *give_up_log_level* parameter. + *stats_base_key* is a string to be used as the base key for the retry-related job stats """ diff --git a/scrapy/item.py b/scrapy/item.py index 4d99ea79d..57d2272e2 100644 --- a/scrapy/item.py +++ b/scrapy/item.py @@ -100,7 +100,7 @@ class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta): #: come first, followed by fields declared in subclasses, and a field #: redefined in a subclass keeps the position of its first definition. #: - #: .. versionchanged:: VERSION + #: .. versionchanged:: 2.17.0 #: Fields are now returned in definition order rather than alphabetical #: order. fields: dict[str, Field] diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 2efe7cd8e..3fb741d7a 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -235,7 +235,7 @@ class LxmlLinkExtractor: can be combined with ``tags='*'`` to consider every tag except a few. Defaults to ``()`` (no tag is excluded). - .. versionadded:: VERSION + .. versionadded:: 2.17.0 :type deny_tags: str or list :param deny_attrs: an attribute or a list of attributes that should not be @@ -244,7 +244,7 @@ class LxmlLinkExtractor: every attribute except a few. Defaults to ``()`` (no attribute is excluded). - .. versionadded:: VERSION + .. versionadded:: 2.17.0 :type deny_attrs: str or list :param canonicalize: canonicalize each extracted url (using