Merge remote-tracking branch 'origin/master' into extras

.pre-commit-config.yaml

@@ -27,6 +27,6 @@ repos:
   hooks:
     - id: sphinx-lint
 - repo: https://github.com/scrapy/sphinx-scrapy
-  rev: 0.8.7
+  rev: 0.8.8
   hooks:
     - id: sphinx-scrapy

conftest.py

@@ -74,40 +74,19 @@ def mockserver() -> Generator[MockServer]:
 
 
 @pytest.fixture  # function scope because it modifies os.environ
-def mitm_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
-    proxy = MitmProxy()
+def proxy_server(
+    request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch
+) -> Generator[str]:
+    kind = request.param
+    proxy = MitmProxy(mode="socks5" if kind == "socks5" else None)
     url = proxy.start()
+    if kind == "https":
+        url = url.replace("http://", "https://")
     monkeypatch.setenv("http_proxy", url)
     monkeypatch.setenv("https_proxy", url)
 
     try:
-        yield proxy
-    finally:
-        proxy.stop()
-
-
-@pytest.fixture  # function scope because it modifies os.environ
-def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
-    proxy = MitmProxy()
-    url = proxy.start().replace("http://", "https://")
-    monkeypatch.setenv("http_proxy", url)
-    monkeypatch.setenv("https_proxy", url)
-
-    try:
-        yield proxy
-    finally:
-        proxy.stop()
-
-
-@pytest.fixture  # function scope because it modifies os.environ
-def socks5_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
-    proxy = MitmProxy(mode="socks5")
-    url = proxy.start()
-    monkeypatch.setenv("http_proxy", url)
-    monkeypatch.setenv("https_proxy", url)
-
-    try:
-        yield proxy
+        yield kind
     finally:
         proxy.stop()
 

docs/conf.py

@@ -159,6 +159,7 @@ scrapy_intersphinx_enable = [
     "parsel",
     "pytest",
     "pypug",
+    "scrapy-lint",
     "sphinx",
     "tox",
     "twisted",

docs/index.rst

@@ -151,6 +151,7 @@ Solving specific problems
    topics/debug
    topics/contracts
    topics/practices
+   topics/security
    topics/broad-crawls
    topics/developer-tools
    topics/dynamic-content
@@ -175,6 +176,10 @@ Solving specific problems
 :doc:`topics/practices`
     Get familiar with some Scrapy common practices.
 
+:doc:`topics/security`
+    Understand the security implications of Scrapy defaults and how to harden
+    them.
+
 :doc:`topics/broad-crawls`
     Tune Scrapy for crawling a lot domains in parallel.
 

docs/requirements.in

@@ -5,4 +5,4 @@ sphinx
 sphinx-notfound-page
 sphinx-rtd-theme
 sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.7
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8

docs/requirements.txt

@@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0
     # via
     #   -r docs/requirements.in
     #   sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@568cf4279dd1f7da4068fa291c8725bab57cfe98
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737
     # via -r docs/requirements.in
 sphinx-sitemap==2.9.0
     # via sphinx-scrapy

docs/topics/asyncio.rst

@@ -4,19 +4,27 @@
 asyncio
 =======
 
-Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
-asyncio reactor <install-asyncio>`, you may use :mod:`asyncio` and
-:mod:`asyncio`-powered libraries in any :doc:`coroutine <coroutines>`.
+Scrapy supports :mod:`asyncio` natively. New projects created with
+:command:`scrapy startproject` have asyncio enabled by default, and you can use
+:mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine
+<coroutines>`.
+
+The rest of this page covers advanced topics. If you are starting a new project,
+no additional setup is needed.
 
 
 .. _install-asyncio:
 
-Installing the asyncio reactor
-==============================
+Configuring the asyncio reactor
+===============================
 
-To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs
-to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``,
-which is the default value.
+New projects generated with :command:`scrapy startproject` have the asyncio
+reactor configured by default. No manual setup is needed.
+
+The :setting:`TWISTED_REACTOR` setting controls which Twisted reactor Scrapy
+uses. Its default value is
+``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, which enables
+:mod:`asyncio` support.
 
 If you are using :class:`~scrapy.crawler.AsyncCrawlerRunner` or
 :class:`~scrapy.crawler.CrawlerRunner`, you also need to

docs/topics/coroutines.rst

@@ -267,6 +267,6 @@ You can also send multiple requests in parallel:
             responses = await asyncio.gather(*tasks)
             yield {
                 "h1": response.css("h1::text").get(),
-                "price": responses[0][1].css(".price::text").get(),
-                "price2": responses[1][1].css(".color::text").get(),
+                "price": responses[0].css(".price::text").get(),
+                "price2": responses[1].css(".color::text").get(),
             }

docs/topics/download-handlers.rst

@@ -39,6 +39,10 @@ for additional schemes and to replace or disable default ones:
         "sftp": "my.download_handlers.SftpHandler",
     }
 
+.. seealso:: :ref:`security-unencrypted-protocols` and
+    :ref:`security-local-resources`, for the security implications of the
+    default ``http``, ``ftp``, ``file`` and ``data`` handlers.
+
 Replacing HTTP(S) download handlers
 -----------------------------------
 

docs/topics/downloader-middleware.rst

@@ -307,26 +307,15 @@ HttpAuthMiddleware
 
 .. class:: HttpAuthMiddleware
 
-    This middleware authenticates all requests generated from certain spiders
-    using `Basic access authentication`_ (aka. HTTP auth).
+    This middleware authenticates requests using `Basic access authentication`_
+    (aka. HTTP auth).
 
-    To enable HTTP authentication for a spider, set the ``http_user`` and
-    ``http_pass`` spider attributes to the authentication data and the
-    ``http_auth_domain`` spider attribute to the domain which requires this
-    authentication (its subdomains will be also handled in the same way).
-    You can set ``http_auth_domain`` to ``None`` to enable the
-    authentication for all requests but you risk leaking your authentication
-    credentials to unrelated domains.
+    Use the :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS`, and
+    :setting:`HTTPAUTH_DOMAIN` settings to configure it. You can also override
+    the credentials per request via :attr:`~scrapy.Request.meta` keys
+    :reqmeta:`http_user`, :reqmeta:`http_pass`, and :reqmeta:`http_auth_domain`.
 
-    .. warning::
-        In previous Scrapy versions HttpAuthMiddleware sent the authentication
-        data with all requests, which is a security problem if the spider
-        makes requests to several different domains. Currently if the
-        ``http_auth_domain`` attribute is not set, the middleware will use the
-        domain of the first request, which will work for some spiders but not
-        for others. In the future the middleware will produce an error instead.
-
-    Example:
+    Example using settings (e.g. in :attr:`~scrapy.Spider.custom_settings`):
 
     .. code-block:: python
 
@@ -334,13 +323,64 @@ HttpAuthMiddleware
 
 
         class SomeIntranetSiteSpider(CrawlSpider):
-            http_user = "someuser"
-            http_pass = "somepass"
-            http_auth_domain = "intranet.example.com"
             name = "intranet.example.com"
+            custom_settings = {
+                "HTTPAUTH_USER": "someuser",
+                "HTTPAUTH_PASS": "somepass",
+                "HTTPAUTH_DOMAIN": "intranet.example.com",
+            }
 
             # .. rest of the spider code omitted ...
 
+    Example using per-request meta:
+
+    .. code-block:: python
+
+        async def start(self):
+            yield Request(
+                "https://intranet.example.com/protected/",
+                meta={
+                    "http_user": "someuser",
+                    "http_pass": "somepass",
+                    "http_auth_domain": "intranet.example.com",
+                },
+            )
+
+.. setting:: HTTPAUTH_USER
+
+HTTPAUTH_USER
+~~~~~~~~~~~~~
+
+Default: ``""``
+
+The username to use for HTTP basic authentication, applied to all requests
+whose URL matches :setting:`HTTPAUTH_DOMAIN`.
+
+.. setting:: HTTPAUTH_PASS
+
+HTTPAUTH_PASS
+~~~~~~~~~~~~~
+
+Default: ``""``
+
+The password to use for HTTP basic authentication.
+
+.. setting:: HTTPAUTH_DOMAIN
+
+HTTPAUTH_DOMAIN
+~~~~~~~~~~~~~~~
+
+Default: ``None``
+
+The domain (and its subdomains) to which HTTP basic authentication credentials
+are sent. Set to ``None`` to send credentials with all requests, but be aware
+that this risks leaking credentials to unrelated domains.
+
+This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER`
+or :setting:`HTTPAUTH_PASS` is set.
+
+.. seealso:: :ref:`security-credential-leakage`
+
 .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication
 
 

docs/topics/extensions.rst

@@ -149,8 +149,6 @@ The following stats are collected:
   (e.g. ``item_dropped_reasons_count/DropItem``).
 * ``response_received_count``: total number of HTTP responses received.
 
-.. _topics-extensions-ref-telnetconsole:
-
 Log Count extension
 ~~~~~~~~~~~~~~~~~~~
 
@@ -159,6 +157,8 @@ Log Count extension
 
 .. autoclass:: LogCount
 
+.. _topics-extensions-ref-telnetconsole:
+
 Telnet console extension
 ~~~~~~~~~~~~~~~~~~~~~~~~
 

docs/topics/item-pipeline.rst

@@ -57,6 +57,8 @@ Any of these methods may be defined as a coroutine function (``async def``).
 Item pipeline example
 =====================
 
+.. _price-pipeline-example:
+
 Price validation and dropping items with no prices
 --------------------------------------------------
 
@@ -246,6 +248,8 @@ returns multiples items with the same id:
                 return item
 
 
+.. _activating-item-pipeline:
+
 Activating an Item Pipeline component
 =====================================
 
@@ -262,3 +266,110 @@ To activate an Item Pipeline component you must add its class to the
 The integer values you assign to classes in this setting determine the
 order in which they run: items go through from lower valued to higher
 valued classes. It's customary to define these numbers in the 0-1000 range.
+
+A complete example
+==================
+
+The examples above show item pipeline components on their own. In a project, a
+pipeline is one of four pieces that work together: the :ref:`item
+<topics-items>` your spider produces, the :ref:`spider <topics-spiders>` that
+yields it, the pipeline that processes it, and the :setting:`ITEM_PIPELINES`
+setting that enables the pipeline.
+
+The following example wires those pieces together to validate the price of
+books scraped from `books.toscrape.com`_, reusing the ``PricePipeline`` from
+:ref:`price-pipeline-example` above.
+
+Define the item in ``myproject/items.py``:
+
+.. code-block:: python
+
+    from dataclasses import dataclass
+
+
+    @dataclass
+    class BookItem:
+        title: str
+        price: float
+
+Yield instances of that item from your spider, e.g. in
+``myproject/spiders/books.py``:
+
+.. skip: next
+.. code-block:: python
+
+    import scrapy
+
+    from myproject.items import BookItem
+
+
+    class BooksSpider(scrapy.Spider):
+        name = "books"
+        start_urls = ["https://books.toscrape.com/"]
+
+        def parse(self, response):
+            for book in response.css("article.product_pod"):
+                yield BookItem(
+                    title=book.css("h3 a::attr(title)").get(),
+                    price=float(book.css("p.price_color::text").re_first(r"[\d.]+")),
+                )
+
+Put the ``PricePipeline`` shown earlier in ``myproject/pipelines.py``, and
+enable it in ``myproject/settings.py``:
+
+.. code-block:: python
+
+    ITEM_PIPELINES = {
+        "myproject.pipelines.PricePipeline": 300,
+    }
+
+With these pieces in place, every ``BookItem`` that ``BooksSpider`` yields
+passes through ``PricePipeline`` before it reaches the :ref:`feed exports
+<topics-feed-exports>` or any other output.
+
+.. _books.toscrape.com: https://books.toscrape.com/
+
+
+Common pitfalls
+===============
+
+The pipeline does not run
+-------------------------
+
+A pipeline component only runs if its class is listed in the
+:setting:`ITEM_PIPELINES` setting, normally in your project's
+:file:`settings.py` file (see :ref:`activating-item-pipeline`). Adding it to
+the spider or elsewhere has no effect.
+
+To confirm that Scrapy loaded your pipeline, look for a line like this near the
+start of the crawl log::
+
+    [scrapy.middleware] INFO: Enabled item pipelines:
+    ['myproject.pipelines.PricePipeline']
+
+If your pipeline is missing from that list, check that its import path matches
+the :setting:`ITEM_PIPELINES` entry, and that the setting is not being
+overridden, for example by :attr:`~scrapy.Spider.custom_settings` or by a
+redefinition of :setting:`ITEM_PIPELINES` in :file:`settings.py`.
+
+The item is not returned
+------------------------
+
+:meth:`process_item` must return the item (or raise
+:exc:`~scrapy.exceptions.DropItem`). A common mistake is to modify the item but
+forget to return it:
+
+.. code-block:: python
+
+    def process_item(self, item):
+        ItemAdapter(item)["price"] *= 1.15
+        # Bug: returns None, so the next component gets None instead of the item.
+
+Return the item so that the next component, and the rest of Scrapy, can keep
+processing it:
+
+.. code-block:: python
+
+    def process_item(self, item):
+        ItemAdapter(item)["price"] *= 1.15
+        return item

docs/topics/jobs.rst

@@ -104,6 +104,15 @@ If you wish to log the requests that couldn't be serialized, you can set the
 :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
 It is ``False`` by default.
 
+.. note:: Because requests are serialized with :mod:`pickle`, the objects you
+    store on a request, such as the values of its
+    :attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta`
+    dictionaries, are deep-copied when the request is written to and later read
+    back from the job directory. As a result, the callback receives a *copy* of
+    those objects rather than the original ones, and changes made to the copy are
+    not reflected in the original object. Keep this in mind if you rely on
+    sharing mutable state through ``cb_kwargs`` or ``meta``.
+
 .. _job-dir-contents:
 
 Job directory contents

docs/topics/media-pipeline.rst

@@ -770,4 +770,28 @@ To enable your custom media pipeline component you must add its class import pat
 
    ITEM_PIPELINES = {"myproject.pipelines.MyImagesPipeline": 300}
 
+Content-based image filtering pipeline
+--------------------------------------
+
+This example overrides ``get_images()`` to filter images using a classifier,
+such as a TensorFlow_ model. Override ``is_valid_image()`` with your
+classification logic:
+
+.. code-block:: python
+
+    from scrapy.pipelines.images import ImagesPipeline, ImageException
+
+
+    class ImageClassifierPipeline(ImagesPipeline):
+        def is_valid_image(self, image):
+            raise NotImplementedError
+
+        def get_images(self, response, request, info, *, item=None):
+            for path, image, buf in super().get_images(response, request, info, item=item):
+                if not self.is_valid_image(image):
+                    raise ImageException("Image does not match criteria")
+                yield path, image, buf
+
+
 .. _MD5 hash: https://en.wikipedia.org/wiki/MD5
+.. _TensorFlow: https://tensorflow.org

docs/topics/practices.rst

@@ -347,10 +347,10 @@ finishes before starting the next one:
     install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
     react(deferred_f_from_coro_f(crawl))
 
-.. note:: When running multiple spiders in the same process, :ref:`reactor
-    settings <reactor-settings>` should not have a different value per spider.
-    Also, :ref:`pre-crawler settings <pre-crawler-settings>` cannot be defined
-    per spider.
+.. note:: When running multiple spiders in the same process, :ref:`logging
+    settings <logging-settings>` and :ref:`reactor settings <reactor-settings>`
+    should not have a different value per spider, and :ref:`pre-crawler
+    settings <pre-crawler-settings>` cannot be defined per spider.
 
 .. seealso:: :ref:`run-from-script`.
 
@@ -387,6 +387,26 @@ crawl::
     curl http://scrapy2.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=2
     curl http://scrapy3.mycompany.com:6800/schedule.json -d project=myproject -d spider=spider1 -d part=3
 
+.. _large-project-startup:
+
+Reducing startup time in large projects
+=======================================
+
+When running a spider with ``scrapy crawl``, Scrapy loads all modules listed in
+:setting:`SPIDER_MODULES` to find the target spider. In large projects with
+many spiders, this can noticeably increase startup time and memory usage.
+
+To avoid loading every spider module, override :setting:`SPIDER_MODULES` on the
+command line to point only to the module that contains the spider you want to
+run:
+
+.. code-block:: shell
+
+    scrapy crawl myspider -s SPIDER_MODULES=myproject.spiders.myspider
+
+Because :setting:`SPIDER_MODULES` is a list setting, you can include multiple
+modules by separating them with commas.
+
 .. _bans:
 
 Avoiding getting banned
@@ -420,6 +440,14 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
 If you are still unable to prevent your bot getting banned, consider contacting
 `commercial support`_.
 
+.. _static-analysis:
+
+Static analysis
+===============
+
+Consider using :doc:`scrapy-lint <scrapy-lint:index>`, a linter for Scrapy
+projects that detects common mistakes and anti-patterns.
+
 .. _Tor project: https://www.torproject.org/
 .. _commercial support: https://www.scrapy.org/companies
 .. _ProxyMesh: https://proxymesh.com/

docs/topics/request-response.rst

@@ -63,7 +63,7 @@ Request objects
 
         .. invisible-code-block: python
 
-            from scrapy.http import Request
+            from scrapy import Request
 
         1. Using a dict:
 
@@ -188,6 +188,13 @@ Request objects
         ``failure.request.cb_kwargs`` in the request's errback. For more information,
         see :ref:`errback-cb_kwargs`.
 
+        .. note:: When :setting:`JOBDIR` is set, requests are serialized to disk
+            with :mod:`pickle` (see :ref:`request-serialization`). As a result,
+            the callback receives a deep copy of any object stored in
+            ``cb_kwargs``, so mutating such an object in the callback does not
+            affect the original. Avoid relying on shared mutable state passed
+            through ``cb_kwargs`` in that case.
+
     .. attribute:: Request.meta
        :value: {}
 
@@ -717,6 +724,9 @@ Those are:
 * :reqmeta:`give_up_log_level`
 * :reqmeta:`handle_httpstatus_all`
 * :reqmeta:`handle_httpstatus_list`
+* :reqmeta:`http_auth_domain`
+* :reqmeta:`http_pass`
+* :reqmeta:`http_user`
 * :reqmeta:`is_start_request`
 * :reqmeta:`max_retry_times`
 * :reqmeta:`proxy`
@@ -799,6 +809,27 @@ give_up_log_level
 :ref:`Logging level <levels>` used for the message logged when a request
 exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
 
+.. reqmeta:: http_auth_domain
+
+http_auth_domain
+----------------
+
+Overrides :setting:`HTTPAUTH_DOMAIN` for this request.
+
+.. reqmeta:: http_pass
+
+http_pass
+---------
+
+Overrides :setting:`HTTPAUTH_PASS` for this request.
+
+.. reqmeta:: http_user
+
+http_user
+---------
+
+Overrides :setting:`HTTPAUTH_USER` for this request.
+
 .. reqmeta:: max_retry_times
 
 max_retry_times
@@ -880,6 +911,11 @@ Request subclasses
 Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass
 it to implement your own custom functionality.
 
+FormRequest
+-----------
+
+.. autoclass:: scrapy.FormRequest
+
 JsonRequest
 -----------
 
@@ -979,7 +1015,7 @@ Response objects
 
         A dictionary-like (:class:`scrapy.http.headers.Headers`) object which contains
         the response headers. Values can be accessed using
-        :meth:`~scrapy.http.headers.Headers.get` to return the first header value with
+        :meth:`~scrapy.http.headers.Headers.get` to return the last header value with
         the specified name or :meth:`~scrapy.http.headers.Headers.getlist` to return
         all header values with the specified name. For example, this call will give you
         all cookies in the headers::

docs/topics/security.rst

@@ -0,0 +1,207 @@
+.. _security:
+
+========
+Security
+========
+
+Scrapy defaults are optimized for web scraping, not for the security posture
+that you might expect from software that handles untrusted input or runs in a
+shared or exposed environment. Some common security practices are unnecessary
+for many scraping use cases, and a few can even prevent valid ones (for
+example, sites that you must scrape may use misconfigured TLS certificates or
+serve content over unencrypted protocols).
+
+This page highlights the Scrapy defaults that have security implications, so
+that you can make an informed decision about whether to keep them, and explains
+how to harden them along with the trade-offs involved.
+
+.. note::
+
+    None of the options below are silver bullets. Which of them make sense
+    depends on your threat model: whether the URLs you crawl come from trusted
+    sources, whether the machine running Scrapy is exposed to a network you do
+    not control, whether the data you handle is sensitive, and so on.
+
+.. _security-untrusted-responses:
+
+Treat responses as untrusted input
+==================================
+
+Regardless of any setting, remember that response data comes from servers you
+do not control, even when you trust the site you are crawling, as responses may
+be tampered with in transit or the server itself may be compromised.
+
+Never pass response data to functions that can execute code or otherwise act on
+their input in an unsafe way, such as :func:`eval`, :func:`exec`, or
+:func:`pickle.loads`, and be careful when writing response data to paths
+derived from the response itself.
+
+TLS connections
+===============
+
+.. _security-certificate-verification:
+
+Certificate verification
+------------------------
+
+By default Scrapy does **not** verify the TLS certificate of HTTPS servers, as
+controlled by the :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting (default:
+``False``).
+
+This default favors reach over security: many sites that are otherwise fine to
+scrape have expired, self-signed, or otherwise invalid certificates, and
+verifying certificates would make requests to them fail.
+
+If the integrity of the connection matters to you (for example, to detect
+man-in-the-middle attacks), set:
+
+.. code-block:: python
+
+    DOWNLOAD_VERIFY_CERTIFICATES = True
+
+* **Pro:** requests to servers with invalid or untrusted certificates fail
+  instead of silently succeeding, protecting you from some man-in-the-middle
+  attacks.
+
+* **Con:** you can no longer scrape sites with misconfigured certificates
+  without re-disabling verification for them.
+
+.. _security-tls-protocols-ciphers:
+
+Protocol versions and ciphers
+-----------------------------
+
+You can restrict the TLS protocol versions that Scrapy accepts through the
+:setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION`
+settings, e.g. to reject obsolete protocol versions.
+
+By default Scrapy uses the OpenSSL ``DEFAULT`` cipher list
+(:setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`), which favors compatibility and still
+allows some older, weaker ciphers. Set it to ``None`` to instead use the curated
+cipher list of the underlying TLS implementation (Twisted), which excludes weak
+ciphers:
+
+.. code-block:: python
+
+    DOWNLOADER_CLIENT_TLS_CIPHERS = None
+
+* **Pro:** connections that would negotiate a weak cipher fail instead of
+  succeeding.
+
+* **Con:** you can no longer connect to servers that only support the excluded
+  ciphers.
+
+.. _security-unencrypted-protocols:
+
+Unencrypted protocols
+=====================
+
+By default Scrapy enables download handlers for unencrypted protocols, namely
+``http://`` and ``ftp://`` (see :setting:`DOWNLOAD_HANDLERS_BASE`). Data sent
+and received over these protocols, including any credentials, travels in plain
+text and can be read or modified by anyone on the network path.
+
+If you only crawl over encrypted protocols, you can disable the unencrypted
+ones so that no request can accidentally be sent unencrypted:
+
+.. code-block:: python
+
+    DOWNLOAD_HANDLERS = {
+        "http": None,
+        "ftp": None,
+    }
+
+* **Pro:** a misconfigured or maliciously-redirected request cannot leak data
+  over an unencrypted connection, as such requests fail instead.
+
+* **Con:** you can no longer crawl resources that are only available over those
+  protocols.
+
+Note that disabling the ``http`` handler also prevents plain-HTTP requests that
+result from following an ``http://`` redirect or link, which is often the point
+of disabling it.
+
+.. _security-local-resources:
+
+Local and non-network resources
+===============================
+
+By default Scrapy enables download handlers for the ``file://`` and ``data:``
+schemes (see :setting:`DOWNLOAD_HANDLERS_BASE`). The ``file://`` handler reads
+arbitrary files from the local filesystem, limited only by the permissions of
+the process running Scrapy.
+
+This is convenient (for example, to parse a local HTML file), but it is a risk
+if any of the URLs you schedule come from an untrusted source: a crafted
+``file:///etc/passwd`` URL could read local files.
+
+If you do not need them, disable these handlers:
+
+.. code-block:: python
+
+    DOWNLOAD_HANDLERS = {
+        "file": None,
+        "data": None,
+    }
+
+* **Pro:** crawled URLs cannot be used to read local files or inline data.
+
+* **Con:** you can no longer fetch ``file://`` or ``data:`` URLs.
+
+More generally, if you crawl URLs from untrusted sources, consider validating
+their schemes (and, where applicable, their hosts) before scheduling requests,
+to avoid server-side request forgery (SSRF) and similar issues.
+
+.. _security-telnet:
+
+Telnet console
+==============
+
+Scrapy enables the :ref:`telnet console <topics-telnetconsole>` by default
+(:setting:`TELNETCONSOLE_ENABLED`). The telnet console is a Python shell
+running inside the Scrapy process, so anyone who can connect to it can run
+arbitrary code in that process.
+
+By default the console binds to ``127.0.0.1`` (:setting:`TELNETCONSOLE_HOST`)
+and is protected by a username (:setting:`TELNETCONSOLE_USERNAME`, default
+``scrapy``) and an automatically generated password
+(:setting:`TELNETCONSOLE_PASSWORD`), so it is only reachable from the local
+machine.
+
+.. warning::
+
+    Telnet does not provide any transport-layer security, so the
+    username/password authentication does not protect the credentials or the
+    session from anyone able to observe the traffic. Never expose the telnet
+    console over an untrusted network by changing :setting:`TELNETCONSOLE_HOST`
+    to a non-local address.
+
+If you do not use the telnet console, disable it entirely:
+
+.. code-block:: python
+
+    TELNETCONSOLE_ENABLED = False
+
+* **Pro:** removes a local code-execution surface and one less listening port.
+
+* **Con:** you can no longer :ref:`inspect and control a running crawler
+  <topics-telnetconsole>` through it.
+
+.. _security-credential-leakage:
+
+Credential leakage across domains
+=================================
+
+Some Scrapy features attach credentials or other sensitive headers to requests,
+and a crawl that spans multiple domains can leak them to unintended hosts:
+
+* HTTP authentication credentials set through
+  :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware` are only
+  sent to the domain set in :setting:`HTTPAUTH_DOMAIN`. Leave this set to the
+  intended domain rather than ``None`` so that credentials are not sent to
+  every domain you crawl.
+
+* The ``Referer`` header may disclose the URLs you crawl to other sites. The
+  default :setting:`REFERRER_POLICY` already avoids sending the referrer from
+  HTTPS to HTTP, but you can tighten it further (for example, to
+  ``same-origin`` or ``no-referrer``) if needed.

docs/topics/settings.rst

@@ -361,6 +361,31 @@ All of these settings, except for :setting:`ASYNCIO_EVENT_LOOP`, are only used
 when the Twisted reactor is used, i.e. when :setting:`TWISTED_REACTOR_ENABLED`
 is ``True``.
 
+.. _logging-settings:
+
+Logging settings
+----------------
+
+**Logging settings** are settings that configure the global root logging
+handler installed by :func:`~scrapy.utils.log.configure_logging`.
+
+These settings can be defined from a spider. However, because only 1 root
+logging handler is active per process, these settings cannot use a different
+value per spider when :ref:`running multiple spiders in the same process
+<run-multiple-spiders>`.
+
+These settings are:
+
+-   :setting:`LOG_DATEFORMAT`
+-   :setting:`LOG_ENABLED`
+-   :setting:`LOG_ENCODING`
+-   :setting:`LOG_FILE`
+-   :setting:`LOG_FILE_APPEND`
+-   :setting:`LOG_FORMAT`
+-   :setting:`LOG_LEVEL`
+-   :setting:`LOG_SHORT_NAMES`
+-   :setting:`LOG_STDOUT`
+
 .. _topics-settings-ref:
 
 Built-in settings reference
@@ -481,6 +506,8 @@ Note that the event loop class must inherit from :class:`asyncio.AbstractEventLo
     :func:`asyncio.set_event_loop`, which will set the specified event loop
     as the current loop for the current OS thread.
 
+.. note:: This is a :ref:`reactor setting <reactor-settings>`.
+
 .. setting:: BOT_NAME
 
 BOT_NAME
@@ -662,6 +689,8 @@ Whether to enable DNS in-memory cache.
     :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
     either when :setting:`DNS_RESOLVER` is set to a different resolver.
 
+.. note:: This is a :ref:`reactor setting <reactor-settings>`.
+
 .. setting:: DNSCACHE_SIZE
 
 DNSCACHE_SIZE
@@ -671,6 +700,8 @@ Default: ``10000``
 
 DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`.
 
+.. note:: This is a :ref:`reactor setting <reactor-settings>`.
+
 .. setting:: TWISTED_DNS_RESOLVER
 
 TWISTED_DNS_RESOLVER
@@ -688,6 +719,8 @@ take the :setting:`DNS_TIMEOUT` setting into account.
 .. note::
     This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
 
+.. note:: This is a :ref:`reactor setting <reactor-settings>`.
+
 .. setting:: DNS_TIMEOUT
 
 DNS_TIMEOUT
@@ -703,6 +736,8 @@ Timeout for processing of DNS queries in seconds. Float is supported.
     :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
     either when :setting:`DNS_RESOLVER` is set to a different resolver.
 
+.. note:: This is a :ref:`reactor setting <reactor-settings>`.
+
 .. setting:: DOWNLOADER
 
 DOWNLOADER
@@ -728,14 +763,18 @@ necessary to access certain HTTPS websites: for example, you may need to use
 ``'DEFAULT:!DH'`` for a website with weak DH parameters or enable a
 specific cipher that is not included in ``DEFAULT`` if a website requires it.
 
+Set this setting to ``None`` to use the default ciphers of the underlying TLS
+implementation.
+
 .. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format
 
 .. note::
 
     Handling of this setting needs to be implemented inside the :ref:`download
     handler <topics-download-handlers>`, so it's not guaranteed to be supported
-    by all 3rd-party handlers. It's currently unsupported by
-    :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
+    by all 3rd-party handlers.
+
+.. seealso:: :ref:`security-tls-protocols-ciphers`
 
 .. setting:: DOWNLOAD_TLS_MAX_VERSION
 
@@ -770,6 +809,8 @@ modern environments.
     by all 3rd-party handlers. Additionally, the set of supported TLS versions
     depends on the TLS implementation being used by the handler.
 
+.. seealso:: :ref:`security-tls-protocols-ciphers`
+
 .. setting:: DOWNLOAD_TLS_MIN_VERSION
 
 DOWNLOAD_TLS_MIN_VERSION
@@ -782,6 +823,8 @@ be used by Scrapy.
 
 See :setting:`DOWNLOAD_TLS_MAX_VERSION` for the details and limitations.
 
+.. seealso:: :ref:`security-tls-protocols-ciphers`
+
 .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
 
 DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
@@ -942,6 +985,9 @@ enabled in your project.
 
 See :setting:`DOWNLOAD_HANDLERS_BASE` for example format.
 
+.. seealso:: :ref:`security-unencrypted-protocols` and
+    :ref:`security-local-resources`
+
 .. setting:: DOWNLOAD_HANDLERS_BASE
 
 DOWNLOAD_HANDLERS_BASE
@@ -989,6 +1035,9 @@ handler (without replacement), place this in your ``settings.py``:
         "ftp": None,
     }
 
+.. seealso:: :ref:`security-unencrypted-protocols` and
+    :ref:`security-local-resources`
+
 
 .. setting:: DOWNLOAD_SLOTS
 
@@ -1147,6 +1196,8 @@ when making a request and abort the request if the verification fails.
     certificate problems are logged when this setting is set to ``False``)
     depends on its implementation.
 
+.. seealso:: :ref:`security-certificate-verification`
+
 .. setting:: DUPEFILTER_CLASS
 
 DUPEFILTER_CLASS
@@ -1443,6 +1494,8 @@ Default: ``True``
 
 Whether to enable logging.
 
+.. note:: This is a :ref:`logging setting <logging-settings>`.
+
 .. setting:: LOG_ENCODING
 
 LOG_ENCODING
@@ -1452,6 +1505,8 @@ Default: ``'utf-8'``
 
 The encoding to use for logging.
 
+.. note:: This is a :ref:`logging setting <logging-settings>`.
+
 .. setting:: LOG_FILE
 
 LOG_FILE
@@ -1461,6 +1516,8 @@ Default: ``None``
 
 File name to use for logging output. If ``None``, standard error will be used.
 
+.. note:: This is a :ref:`logging setting <logging-settings>`.
+
 .. setting:: LOG_FILE_APPEND
 
 LOG_FILE_APPEND
@@ -1471,6 +1528,8 @@ Default: ``True``
 If ``False``, the log file specified with :setting:`LOG_FILE` will be
 overwritten (discarding the output from previous runs, if any).
 
+.. note:: This is a :ref:`logging setting <logging-settings>`.
+
 .. setting:: LOG_FORMAT
 
 LOG_FORMAT
@@ -1482,6 +1541,8 @@ String for formatting log messages. Refer to the
 :ref:`Python logging documentation <logrecord-attributes>` for the whole
 list of available placeholders.
 
+.. note:: This is a :ref:`logging setting <logging-settings>`.
+
 .. setting:: LOG_DATEFORMAT
 
 LOG_DATEFORMAT
@@ -1494,6 +1555,8 @@ in :setting:`LOG_FORMAT`. Refer to the
 :ref:`Python datetime documentation <strftime-strptime-behavior>` for the
 whole list of available directives.
 
+.. note:: This is a :ref:`logging setting <logging-settings>`.
+
 .. setting:: LOG_FORMATTER
 
 LOG_FORMATTER
@@ -1513,6 +1576,8 @@ Default: ``'DEBUG'``
 Minimum level to log. Available levels are: CRITICAL, ERROR, WARNING,
 INFO, DEBUG. For more info see :ref:`topics-logging`.
 
+.. note:: This is a :ref:`logging setting <logging-settings>`.
+
 .. setting:: LOG_STDOUT
 
 LOG_STDOUT
@@ -1524,6 +1589,8 @@ If ``True``, all standard output (and error) of your process will be redirected
 to the log. For example if you ``print('hello')`` it will appear in the Scrapy
 log.
 
+.. note:: This is a :ref:`logging setting <logging-settings>`.
+
 .. setting:: LOG_SHORT_NAMES
 
 LOG_SHORT_NAMES
@@ -1534,6 +1601,8 @@ Default: ``False``
 If ``True``, the logs will just contain the root path. If it is set to ``False``
 then it displays the component responsible for the log output
 
+.. note:: This is a :ref:`logging setting <logging-settings>`.
+
 .. setting:: LOG_VERSIONS
 
 LOG_VERSIONS
@@ -1696,6 +1765,8 @@ multi-purpose thread pool used by various Scrapy components. Threaded
 DNS Resolver, BlockingFeedStorage, S3FilesStore just to name a few. Increase
 this value if you're experiencing problems with insufficient blocking IO.
 
+.. note:: This is a :ref:`reactor setting <reactor-settings>`.
+
 .. setting:: REDIRECT_PRIORITY_ADJUST
 
 REDIRECT_PRIORITY_ADJUST
@@ -1922,6 +1993,8 @@ Default: ``'scrapy.spiderloader.SpiderLoader'``
 The class that will be used for loading spiders, which must implement the
 :ref:`topics-api-spiderloader`.
 
+.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
+
 .. setting:: SPIDER_LOADER_WARN_ONLY
 
 SPIDER_LOADER_WARN_ONLY
@@ -1934,6 +2007,8 @@ it will fail loudly if there is any ``ImportError`` or ``SyntaxError`` exception
 But you can choose to silence this exception and turn it into a simple
 warning by setting ``SPIDER_LOADER_WARN_ONLY = True``.
 
+.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
+
 .. setting:: SPIDER_MIDDLEWARES
 
 SPIDER_MIDDLEWARES
@@ -1979,6 +2054,8 @@ Example:
 
     SPIDER_MODULES = ["mybot.spiders_prod", "mybot.spiders_dev"]
 
+.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
+
 .. setting:: STATS_CLASS
 
 STATS_CLASS
@@ -2011,6 +2088,8 @@ Default: ``True`` (``False`` when :setting:`TWISTED_REACTOR_ENABLED` is ``False`
 A boolean which specifies if the :ref:`telnet console <topics-telnetconsole>`
 will be enabled (provided its extension is also enabled).
 
+.. seealso:: :ref:`security-telnet`
+
 .. setting:: TEMPLATES_DIR
 
 TEMPLATES_DIR
@@ -2050,7 +2129,7 @@ stopped) will not apply. This mode is currently experimental and may not be
 suitable for production use. It may also not be supported by 3rd-party code.
 See :ref:`asyncio-without-reactor` for more information about this mode.
 
-.. note:: This setting can't be set :ref:`per-spider <spider-settings>`.
+.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
 
 .. versionadded:: 2.15.0
 
@@ -2157,6 +2236,7 @@ current platform.
 
 For additional information, see :doc:`core/howto/choosing-reactor`.
 
+.. note:: This is a :ref:`reactor setting <reactor-settings>`.
 
 .. setting:: URLLENGTH_LIMIT
 

docs/topics/spider-middleware.rst

@@ -355,6 +355,8 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
     using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key,
     with the same acceptable values as for the ``REFERRER_POLICY`` setting.
 
+.. seealso:: :ref:`security-credential-leakage`
+
 Acceptable values for REFERRER_POLICY
 *************************************
 

docs/topics/spiders.rst

@@ -354,11 +354,6 @@ Otherwise, you would cause iteration over a ``start_urls`` string
 (a very common python pitfall)
 resulting in each character being seen as a separate url.
 
-A valid use case is to set the http auth credentials
-used by :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`::
-
-    scrapy crawl myspider -a http_user=myuser -a http_pass=mypassword
-
 Spider arguments can also be passed through the Scrapyd ``schedule.json`` API.
 See `Scrapyd documentation`_.
 

docs/topics/telnetconsole.rst

@@ -29,6 +29,8 @@ disable it if you want. For more information about the extension itself see
 .. note::
     This feature is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
 
+.. seealso:: :ref:`security-telnet`
+
 .. highlight:: none
 
 How to access the telnet console
@@ -94,8 +96,6 @@ convenience:
 +----------------+-------------------------------------------------------------------+
 | ``p``          | a shortcut to the :func:`pprint.pprint` function                  |
 +----------------+-------------------------------------------------------------------+
-| ``hpy``        | for memory debugging (see :ref:`topics-leaks`)                    |
-+----------------+-------------------------------------------------------------------+
 
 Telnet console usage examples
 =============================
@@ -192,6 +192,8 @@ Default: ``'127.0.0.1'``
 
 The interface the telnet console should listen on
 
+.. seealso:: :ref:`security-telnet`
+
 
 .. setting:: TELNETCONSOLE_USERNAME
 

pyproject.toml

@@ -248,6 +248,7 @@ disable = [
     "undefined-variable",
     "unused-argument",
     "unused-variable",
+    "use-implicit-booleaness-not-comparison",
     "useless-import-alias",  # used as a hint to mypy
     "useless-return",  # https://github.com/pylint-dev/pylint/issues/6530
     "wrong-import-position",
@@ -291,6 +292,8 @@ markers = [
 ]
 filterwarnings = [
     "ignore::DeprecationWarning:twisted.web.static",
+    # Twisted doesn't close failed sockets after CannotListenError: https://github.com/twisted/twisted/issues/6108
+    "ignore:Exception ignored in. <socket\\.socket.*laddr=..0\\.0\\.0\\.0., 0.:pytest.PytestUnraisableExceptionWarning",
 ]
 
 [tool.ruff.lint]

scrapy/__main__.py

@@ -1,3 +1,4 @@
+# pragma: no file cover
 from scrapy.cmdline import execute
 
 if __name__ == "__main__":

scrapy/addons.py

@@ -64,7 +64,7 @@ class AddonManager:
 
         :param settings: The :class:`~scrapy.settings.BaseSettings` object from \
             which to read the early add-on configuration
-        :type settings: :class:`~scrapy.settings.Settings`
+        :type settings: :class:`~scrapy.settings.BaseSettings`
         """
         for clspath in build_component_list(settings["ADDONS"]):
             addoncls = load_object(clspath)

scrapy/commands/__init__.py

@@ -16,6 +16,8 @@ from twisted.python import failure
 
 from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
 from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
+from scrapy.utils.deprecate import method_is_overridden
+from scrapy.utils.python import global_object_name
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
@@ -36,6 +38,14 @@ class ScrapyCommand(ABC):
 
     def __init__(self) -> None:
         self.settings: Settings | None = None  # set in scrapy.cmdline
+        if method_is_overridden(self.__class__, ScrapyCommand, "help"):
+            warnings.warn(
+                "The ScrapyCommand.help() method is deprecated and overriding "
+                f"it, as the {global_object_name(self.__class__)} class does, "
+                "has no effect; override long_desc() instead.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
 
     def set_crawler(self, crawler: Crawler) -> None:  # pragma: no cover
         warnings.warn(
@@ -63,15 +73,16 @@ class ScrapyCommand(ABC):
     def long_desc(self) -> str:
         """A long description of the command. Return short description when not
         available. It cannot contain newlines since contents will be formatted
-        by optparser which removes newlines and wraps text.
+        by argparse which removes newlines and wraps text.
         """
         return self.short_desc()
 
     def help(self) -> str:
-        """An extensive help for the command. It will be shown when using the
-        "help" command. It can contain newlines since no post-formatting will
-        be applied to its contents.
-        """
+        warnings.warn(
+            "ScrapyCommand.help() is deprecated, use long_desc() instead.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         return self.long_desc()
 
     def add_options(self, parser: argparse.ArgumentParser) -> None:

scrapy/commands/edit.py

@@ -1,12 +1,29 @@
-import argparse
+from __future__ import annotations
+
 import os
+import shlex
+import subprocess
 import sys
-from typing import Any, ClassVar
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, ClassVar
 
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.spiderloader import get_spider_loader
 
+if TYPE_CHECKING:
+    import argparse
+
+
+def _edit_file(editor: str, file_path: str | os.PathLike[str]) -> int:
+    """Open ``file_path`` with ``editor`` and return the editor exit code.
+
+    ``editor`` may include arguments (e.g. ``"code -w"``); it is split with
+    :func:`shlex.split` and the file is passed as a separate argument, so no
+    shell is involved.
+    """
+    return subprocess.call([*shlex.split(editor), os.fspath(file_path)])  # noqa: S603
+
 
 class Command(ScrapyCommand):
     requires_project = True
@@ -45,4 +62,4 @@ class Command(ScrapyCommand):
         sfile = sys.modules[spidercls.__module__].__file__
         assert sfile
         sfile = sfile.replace(".pyc", ".py")
-        self.exitcode = os.system(f'{editor} "{sfile}"')  # noqa: S605
+        self.exitcode = _edit_file(editor, Path(sfile))

scrapy/commands/genspider.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import os
 import shutil
 import string
 from importlib import import_module
@@ -10,12 +9,14 @@ from urllib.parse import urlparse
 
 import scrapy
 from scrapy.commands import ScrapyCommand
+from scrapy.commands.edit import _edit_file
 from scrapy.exceptions import UsageError
 from scrapy.spiderloader import get_spider_loader
 from scrapy.utils.template import render_templatefile, string_camelcase
 
 if TYPE_CHECKING:
     import argparse
+    import os
 
 
 def sanitize_module_name(module_name: str) -> str:
@@ -118,9 +119,11 @@ class Command(ScrapyCommand):
 
         template_file = self._find_template(opts.template)
         if template_file:
-            self._genspider(module, name, url, opts.template, template_file)
+            spider_file = self._genspider(
+                module, name, url, opts.template, template_file
+            )
             if opts.edit:
-                self.exitcode = os.system(f'scrapy edit "{name}"')  # noqa: S605
+                self.exitcode = _edit_file(self.settings["EDITOR"], spider_file)
 
     def _generate_template_variables(
         self,
@@ -148,7 +151,7 @@ class Command(ScrapyCommand):
         url: str,
         template_name: str,
         template_file: str | os.PathLike[str],
-    ) -> None:
+    ) -> Path:
         """Generate the spider module, based on the given template"""
         assert self.settings is not None
         tvars = self._generate_template_variables(module, name, url, template_name)
@@ -168,6 +171,7 @@ class Command(ScrapyCommand):
         )
         if spiders_module:
             print(f"in module:\n  {spiders_module.__name__}.{module}")
+        return Path(spider_file)
 
     def _find_template(self, template: str) -> Path | None:
         template_file = Path(self.templates_dir, f"{template}.tmpl")

scrapy/contracts/__init__.py

@@ -51,6 +51,8 @@ class Contract:
                     results.addSuccess(self.testcase_pre)
                 cb_result = cb(response, **cb_kwargs)
                 if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
+                    if isinstance(cb_result, CoroutineType):
+                        cb_result.close()
                     raise TypeError("Contracts don't support async callbacks")
                 return list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
 
@@ -67,6 +69,8 @@ class Contract:
             def wrapper(response: Response, **cb_kwargs: Any) -> list[Any]:
                 cb_result = cb(response, **cb_kwargs)
                 if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
+                    if isinstance(cb_result, CoroutineType):
+                        cb_result.close()
                     raise TypeError("Contracts don't support async callbacks")
                 output = list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
                 try:

scrapy/core/downloader/contextfactory.py

@@ -17,7 +17,6 @@ from zope.interface.verify import verifyObject
 
 from scrapy.core.downloader.tls import (
     _TWISTED_VERSION_MAP,
-    DEFAULT_CIPHERS,
     _openssl_methods,
     _ScrapyClientTLSOptions,
     _ScrapyClientTLSOptions26,
@@ -48,8 +47,13 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
     instance.
 
     The purpose of this custom class is to provide a ``creatorForNetloc()``
-    method that returns a ``_ScrapyClientTLSOptions`` instance configured based
-    on TLS settings provided to the factory.
+    method that returns:
+
+    - a ``_ScrapyClientTLSOptions26`` or ``_ScrapyClientTLSOptions`` instance
+      configured based on TLS settings provided to the factory (when the
+      certificate verification is disabled);
+    - a result of ``optionsForClientTLS()`` called with those TLS settings
+      (when the certificate verification is enabled).
     """
 
     def __init__(
@@ -68,11 +72,11 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         self.tls_min_version: TLSVersion | None = tls_min_version
         self.tls_max_version: TLSVersion | None = tls_max_version
         self.tls_verbose_logging: bool = tls_verbose_logging  # unused
-        self.tls_ciphers: AcceptableCiphers
-        if tls_ciphers:
-            self.tls_ciphers = AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
-        else:
-            self.tls_ciphers = DEFAULT_CIPHERS
+        self.tls_ciphers: AcceptableCiphers | None = (
+            AcceptableCiphers.fromOpenSSLCipherString(tls_ciphers)
+            if tls_ciphers
+            else None
+        )
         self._verify_certificates = verify_certificates
 
     @classmethod

scrapy/core/downloader/handlers/_base_streaming.py

@@ -241,6 +241,16 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon
             body=response_body.getvalue(),
         )
 
+    @staticmethod
+    def _request_headers(request: Request) -> Headers:
+        """Get a prepared copy of the request headers.
+
+        This removes the Proxy-Authorization header.
+        """
+        headers = request.headers.copy()
+        headers.pop(b"Proxy-Authorization", None)
+        return headers
+
     def _get_bind_address_host(self) -> str | None:
         """Return the host portion of the bind address.
 
@@ -279,10 +289,8 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon
         if not proxy:
             return None, None
         proxy = add_http_if_no_scheme(proxy)
-        auth_header: list[bytes] | None = request.headers.pop(
-            b"Proxy-Authorization", None
-        )
-        return proxy, auth_header[0].decode("ascii") if auth_header else None
+        auth_header: bytes | None = request.headers.get(b"Proxy-Authorization")
+        return proxy, auth_header.decode("ascii") if auth_header else None
 
     def _extract_proxy_url_with_creds(self, request: Request) -> str | None:
         """Return the proxy URL with the userinfo added based on the

scrapy/core/downloader/handlers/_httpx.py

@@ -152,13 +152,14 @@ class HttpxDownloadHandler(_Base):
                 f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed."
             )
         client = self._get_client(proxy)
+        headers = self._request_headers(request).to_tuple_list()
 
         try:
             async with client.stream(
                 request.method,
                 request.url,
                 content=request.body,
-                headers=request.headers.to_tuple_list(),
+                headers=headers,
                 timeout=timeout,
             ) as response:
                 yield response

scrapy/core/downloader/handlers/ftp.py

@@ -2,9 +2,9 @@
 An asynchronous FTP file download handler for scrapy which somehow emulates an http response.
 
 FTP connection parameters are passed using the request meta field:
-- ftp_user (required)
-- ftp_password (required)
-- ftp_passive (by default, enabled) sets FTP connection passive mode
+- ftp_user (optional, falls back to FTP_USER)
+- ftp_password (optional, falls back to FTP_PASSWORD)
+- ftp_passive (optional, falls back to FTP_PASSIVE_MODE) sets FTP connection passive mode
 - ftp_local_filename
         - If not given, file data will come in the response.body, as a normal scrapy Response,
         which will imply that the entire file will be on memory.
@@ -119,7 +119,10 @@ class FTPDownloadHandler(BaseDownloadHandler):
                 httpcode = self.CODE_MAPPING.get(ftpcode, self.CODE_MAPPING["default"])
                 return Response(url=request.url, status=httpcode, body=message.encode())
             raise
-        protocol.close()
+        finally:
+            protocol.close()
+            assert client.transport
+            client.transport.loseConnection()
         headers = {"local filename": protocol.filename or b"", "size": protocol.size}
         body = protocol.filename or protocol.body.read()
         respcls = responsetypes.from_args(url=request.url, body=body)

scrapy/core/downloader/handlers/http11.py

@@ -104,7 +104,6 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
         self._disconnect_timeout: int = 1
 
     async def download_request(self, request: Request) -> Response:
-        """Return a deferred for the HTTP download"""
         if hasattr(self._crawler.spider, "download_maxsize"):  # pragma: no cover
             warn_on_deprecated_spider_attribute("download_maxsize", "DOWNLOAD_MAXSIZE")
         if hasattr(self._crawler.spider, "download_warnsize"):  # pragma: no cover
@@ -283,7 +282,7 @@ def _tunnel_request_data(
 
 
 class _TunnelingAgent(Agent):
-    """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
+    """An agent that uses a ``_TunnelingTCP4ClientEndpoint`` to make HTTPS
     downloads. It may look strange that we have chosen to subclass Agent and not
     ProxyAgent but consider that after the tunnel is opened the proxy is
     transparent to the client; thus the agent should behave like there is no
@@ -545,7 +544,8 @@ class _ScrapyAgent:
                 expected_size, maxsize, request, expected=True
             )
             logger.warning(warning_msg)
-            txresponse._transport.loseConnection()
+            # Abort connection immediately.
+            txresponse._transport._producer.abortConnection()
             raise DownloadCancelledError(warning_msg)
 
         if warnsize and expected_size > warnsize:

scrapy/core/downloader/tls.py

@@ -43,6 +43,13 @@ _openssl_methods: dict[str, int] = {
 
 
 def __getattr__(name: str) -> Any:
+    if name == "DEFAULT_CIPHERS":
+        warnings.warn(
+            "scrapy.core.downloader.tls.DEFAULT_CIPHERS is deprecated.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return AcceptableCiphers.fromOpenSSLCipherString("DEFAULT")
     deprecated = {
         "METHOD_TLS": "TLS",
         "METHOD_TLSv10": "TLSv1.0",
@@ -177,8 +184,3 @@ class _ScrapyClientTLSOptions26(ClientTLSOptions):
             return True
 
         return verifyCallback
-
-
-DEFAULT_CIPHERS: AcceptableCiphers = AcceptableCiphers.fromOpenSSLCipherString(
-    "DEFAULT"
-)

scrapy/core/http2/agent.py

@@ -114,11 +114,7 @@ class H2ConnectionPool:
             d.errback(ResponseFailed(errors))
 
     def close_connections(self) -> None:
-        """Close all the HTTP/2 connections and remove them from pool
-
-        Returns:
-            Deferred that fires when all connections have been closed
-        """
+        """Close all the HTTP/2 connections and remove them from pool."""
         for conn in self._connections.values():
             assert conn.transport is not None  # typing
             conn.transport.abortConnection()

scrapy/core/http2/protocol.py

@@ -101,7 +101,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
                 uri is used to verify that incoming client requests have correct
                 base URL.
             settings -- Scrapy project settings
-            conn_lost_deferred -- Deferred fires with the reason: Failure to notify
+            conn_lost_deferred -- Deferred that fires with the list of underlying exceptions to notify
                 that connection was lost
             tls_verbose_logging -- Whether to log TLS details
         """
@@ -375,7 +375,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
     def _handle_events(self, events: list[Event]) -> None:
         """Private method which acts as a bridge between the events
-        received from the HTTP/2 data and IH2EventsHandler
+        received from the HTTP/2 data and the handlers in this class.
 
         Arguments:
             events -- A list of events that the remote peer triggered by sending data

scrapy/core/scheduler.py

@@ -131,10 +131,10 @@ class Scheduler(BaseScheduler):
     (:setting:`SCHEDULER_PRIORITY_QUEUE`) that sort requests by
     :attr:`~scrapy.http.Request.priority`.
 
-    By default, a single, memory-based priority queue is used for all requests.
-    When using :setting:`JOBDIR`, a disk-based priority queue is also created,
+    By default, memory-based priority queues are used for all requests.
+    When using :setting:`JOBDIR`, disk-based priority queues are also created,
     and only unserializable requests are stored in the memory-based priority
-    queue. For a given priority value, requests in memory take precedence over
+    queues. For a given priority value, requests in memory take precedence over
     requests in disk.
 
     Each priority queue stores requests in separate internal queues, one per
@@ -209,8 +209,8 @@ class Scheduler(BaseScheduler):
     -------------------------
 
     While pending requests are below the configured values of
-    :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
-    or :setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent
+    :setting:`CONCURRENT_REQUESTS` or
+    :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, those requests are sent
     concurrently.
 
     As a result, the first few requests of a crawl may not follow the desired
@@ -342,7 +342,7 @@ class Scheduler(BaseScheduler):
     def open(self, spider: Spider) -> Deferred[None] | None:
         """
         (1) initialize the memory queue
-        (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
+        (2) initialize the disk queue if the ``jobdir`` argument wasn't empty
         (3) return the result of the dupefilter's ``open`` method
         """
         self.spider: Spider = spider

scrapy/core/scraper.py

@@ -441,7 +441,7 @@ class Scraper:
         self, output: Any, response: Response | Failure
     ) -> Deferred[None]:
         """Process each Request/Item (given in the output parameter) returned
-        from the given spider.
+        from the spider.
 
         Items are sent to the item pipelines, requests are scheduled.
         """
@@ -451,7 +451,7 @@ class Scraper:
         self, output: Any, response: Response | Failure
     ) -> None:
         """Process each Request/Item (given in the output parameter) returned
-        from the given spider.
+        from the spider.
 
         Items are sent to the item pipelines, requests are scheduled.
         """

scrapy/crawler.py

@@ -363,9 +363,12 @@ class CrawlerRunnerBase(ABC):
         """
         Return a :class:`~scrapy.crawler.Crawler` object.
 
-        * If ``crawler_or_spidercls`` is a Crawler, it is returned as-is.
+        * If ``crawler_or_spidercls`` is a Crawler, the runner's settings are
+          merged into it as defaults: for each setting, the runner's value
+          is applied only if the Crawler does not already have that setting at
+          an equal or higher priority. The Crawler is then returned.
         * If ``crawler_or_spidercls`` is a Spider subclass, a new Crawler
-          is constructed for it.
+          is constructed for it using this runner's settings.
         * If ``crawler_or_spidercls`` is a string, this function finds
           a spider with this name in a Scrapy project (using spider loader),
           then creates a Crawler instance for it.
@@ -376,6 +379,7 @@ class CrawlerRunnerBase(ABC):
                 "it must be a spider class (or a Crawler object)"
             )
         if isinstance(crawler_or_spidercls, Crawler):
+            crawler_or_spidercls.settings.update(self.settings)
             return crawler_or_spidercls
         return self._create_crawler(crawler_or_spidercls)
 
@@ -527,8 +531,8 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
         """
         Run a crawler with the provided arguments.
 
-        It will call the given Crawler's :meth:`~Crawler.crawl` method, while
-        keeping track of it so it can be stopped later.
+        It will call the given Crawler's :meth:`~Crawler.crawl_async` method,
+        while keeping track of it so it can be stopped later.
 
         If ``crawler_or_spidercls`` isn't a :class:`~scrapy.crawler.Crawler`
         instance, this method will try to create one using this parameter as
@@ -769,7 +773,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):
         """
         This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
         size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS
-        resolver based on :setting:`DNSCACHE_ENABLED`.
+        resolver based on :setting:`TWISTED_DNS_RESOLVER`.
 
         If ``stop_after_crawl`` is True, the reactor will be stopped after all
         crawlers have finished, using :meth:`join`.
@@ -871,10 +875,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
 
         When using a reactor it adjusts its pool size to
         :setting:`REACTOR_THREADPOOL_MAXSIZE` and installs a DNS resolver based
-        on :setting:`DNSCACHE_ENABLED`.
+        on :setting:`TWISTED_DNS_RESOLVER`.
 
-        If ``stop_after_crawl`` is True, the reactor will be stopped after all
-        crawlers have finished, using :meth:`join`.
+        If ``stop_after_crawl`` is True, the reactor/event loop will be stopped
+        after all crawlers have finished, using :meth:`join`.
 
         :param bool stop_after_crawl: stop or not the reactor when all
             crawlers have finished

scrapy/downloadermiddlewares/cookies.py

@@ -12,6 +12,7 @@ from scrapy.http.cookies import CookieJar
 from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
+from scrapy.utils.request import _decode_cookie, _to_verbose_cookies
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Sequence
@@ -134,29 +135,10 @@ class CookiesMiddleware:
         Given a dict consisting of cookie components, return its string representation.
         Decode from bytes if necessary.
         """
-        decoded = {}
+        decoded = _decode_cookie(cookie, request)
+        if decoded is None:
+            return None
         flags = set()
-        for key in ("name", "value", "path", "domain"):
-            value = cookie.get(key)
-            if value is None:
-                if key in {"name", "value"}:
-                    msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
-                    logger.warning(msg)
-                    return None
-                continue
-            if isinstance(value, (bool, float, int, str)):
-                decoded[key] = str(value)
-            else:
-                assert isinstance(value, bytes)
-                try:
-                    decoded[key] = value.decode("utf8")
-                except UnicodeDecodeError:
-                    logger.warning(
-                        "Non UTF-8 encoded cookie found in request %s: %s",
-                        request,
-                        cookie,
-                    )
-                    decoded[key] = value.decode("latin1", errors="replace")
         for flag in ("secure",):
             value = cookie.get(flag, _UNSET)
             if value is _UNSET or not value:
@@ -177,11 +159,7 @@ class CookiesMiddleware:
         """
         if not request.cookies:
             return ()
-        cookies: Iterable[VerboseCookie]
-        if isinstance(request.cookies, dict):
-            cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items())
-        else:
-            cookies = request.cookies
+        cookies: Iterable[VerboseCookie] = _to_verbose_cookies(request.cookies)
         for cookie in cookies:
             cookie.setdefault("secure", urlparse_cached(request).scheme == "https")
         formatted = filter(None, (self._format_cookie(c, request) for c in cookies))

scrapy/downloadermiddlewares/httpauth.py

@@ -6,11 +6,14 @@ See documentation in docs/topics/downloader-middleware.rst
 
 from __future__ import annotations
 
+import warnings
 from typing import TYPE_CHECKING
 
 from w3lib.http import basic_auth_header
 
 from scrapy import Request, Spider, signals
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.settings import SETTINGS_PRIORITIES
 from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.url import url_is_from_any_domain
 
@@ -23,12 +26,28 @@ if TYPE_CHECKING:
 
 
 class HttpAuthMiddleware:
-    """Set Basic HTTP Authorization header
-    (http_user and http_pass spider class attributes)"""
+    """Set Basic HTTP Authorization header."""
+
+    def __init__(self) -> None:
+        self._auth: bytes | None = None
+        self._domain: str | None = None
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
         o = cls()
+        usr = crawler.settings.get("HTTPAUTH_USER", "")
+        pwd = crawler.settings.get("HTTPAUTH_PASS", "")
+        if usr or pwd:
+            domain_priority = crawler.settings.getpriority("HTTPAUTH_DOMAIN") or 0
+            if domain_priority <= SETTINGS_PRIORITIES["default"]:
+                raise ValueError(
+                    "HTTPAUTH_DOMAIN must be set when HTTPAUTH_USER or HTTPAUTH_PASS "
+                    "is configured. Set it to a domain (e.g. 'example.com') to restrict "
+                    "credentials to that domain, or set it to None to send credentials "
+                    "with all requests."
+                )
+            o._auth = basic_auth_header(usr, pwd)
+            o._domain = crawler.settings.get("HTTPAUTH_DOMAIN")
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         return o
 
@@ -36,18 +55,34 @@ class HttpAuthMiddleware:
         usr = getattr(spider, "http_user", "")
         pwd = getattr(spider, "http_pass", "")
         if usr or pwd:
-            self.auth = basic_auth_header(usr, pwd)
-            self.domain = spider.http_auth_domain  # type: ignore[attr-defined]
+            warnings.warn(
+                "Use the HTTPAUTH_USER, HTTPAUTH_PASS, and HTTPAUTH_DOMAIN settings "
+                "instead of the http_user, http_pass, and http_auth_domain spider "
+                "attributes. Support for the spider attributes will be removed in a "
+                "future version of Scrapy.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+            self._auth = basic_auth_header(usr, pwd)
+            self._domain = spider.http_auth_domain  # type: ignore[attr-defined]
 
     @_warn_spider_arg
     def process_request(
         self, request: Request, spider: Spider | None = None
     ) -> Request | Response | None:
-        auth = getattr(self, "auth", None)
-        if (
-            auth
-            and b"Authorization" not in request.headers
-            and (not self.domain or url_is_from_any_domain(request.url, [self.domain]))
+        if b"Authorization" in request.headers:
+            return None
+        # Per-request meta overrides
+        usr = request.meta.get("http_user", "")
+        pwd = request.meta.get("http_pass", "")
+        if usr or pwd:
+            domain = request.meta.get("http_auth_domain")
+            if not domain or url_is_from_any_domain(request.url, [domain]):
+                request.headers[b"Authorization"] = basic_auth_header(usr, pwd)
+            return None
+        # Middleware-level auth
+        if self._auth and (
+            not self._domain or url_is_from_any_domain(request.url, [self._domain])
         ):
-            request.headers[b"Authorization"] = auth
+            request.headers[b"Authorization"] = self._auth
         return None

scrapy/downloadermiddlewares/httpcompression.py

@@ -6,7 +6,7 @@ from logging import getLogger
 from typing import TYPE_CHECKING, Any
 
 from scrapy import Request, Spider, signals
-from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.utils._compression import (
@@ -60,7 +60,7 @@ else:
 
 
 class HttpCompressionMiddleware:
-    """This middleware allows compressed (gzip, deflate) traffic to be
+    """This middleware allows compressed (gzip, deflate etc.) traffic to be
     sent/received from websites"""
 
     def __init__(
@@ -70,6 +70,12 @@ class HttpCompressionMiddleware:
         crawler: Crawler | None = None,
     ):
         if not crawler:
+            warnings.warn(
+                "Instantiating HttpCompressionMiddleware without a 'crawler' "
+                "argument is deprecated.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
             self.stats = stats
             self._max_size = 1073741824
             self._warn_size = 33554432
@@ -108,49 +114,47 @@ class HttpCompressionMiddleware:
     ) -> Request | Response:
         if request.method == "HEAD":
             return response
-        if isinstance(response, Response):
-            content_encoding = response.headers.getlist("Content-Encoding")
-            if content_encoding:
-                max_size = request.meta.get("download_maxsize", self._max_size)
-                warn_size = request.meta.get("download_warnsize", self._warn_size)
-                try:
-                    decoded_body, content_encoding = self._handle_encoding(
-                        response.body, content_encoding, max_size
-                    )
-                except _DecompressionMaxSizeExceeded as e:
-                    raise IgnoreRequest(
-                        f"Ignored response {response} because its body "
-                        f"({len(response.body)} B compressed, "
-                        f"{e.decompressed_size} B decompressed so far) exceeded "
-                        f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
-                    ) from e
-                if len(response.body) < warn_size <= len(decoded_body):
-                    logger.warning(
-                        f"{response} body size after decompression "
-                        f"({len(decoded_body)} B) is larger than the "
-                        f"download warning size ({warn_size} B)."
-                    )
-                if content_encoding:
-                    self._warn_unknown_encoding(response, content_encoding)
-                response.headers["Content-Encoding"] = content_encoding
-                if self.stats:
-                    self.stats.inc_value(
-                        "httpcompression/response_bytes",
-                        len(decoded_body),
-                    )
-                    self.stats.inc_value("httpcompression/response_count")
-                respcls = responsetypes.from_args(
-                    headers=response.headers, url=response.url, body=decoded_body
+        content_encoding = response.headers.getlist("Content-Encoding")
+        if content_encoding:
+            max_size = request.meta.get("download_maxsize", self._max_size)
+            warn_size = request.meta.get("download_warnsize", self._warn_size)
+            try:
+                decoded_body, content_encoding = self._handle_encoding(
+                    response.body, content_encoding, max_size
                 )
-                kwargs: dict[str, Any] = {"body": decoded_body}
-                if issubclass(respcls, TextResponse):
-                    # force recalculating the encoding until we make sure the
-                    # responsetypes guessing is reliable
-                    kwargs["encoding"] = None
-                response = response.replace(cls=respcls, **kwargs)
-                if not content_encoding:
-                    del response.headers["Content-Encoding"]
-
+            except _DecompressionMaxSizeExceeded as e:
+                raise IgnoreRequest(
+                    f"Ignored response {response} because its body "
+                    f"({len(response.body)} B compressed, "
+                    f"{e.decompressed_size} B decompressed so far) exceeded "
+                    f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
+                ) from e
+            if len(response.body) < warn_size <= len(decoded_body):
+                logger.warning(
+                    f"{response} body size after decompression "
+                    f"({len(decoded_body)} B) is larger than the "
+                    f"download warning size ({warn_size} B)."
+                )
+            if content_encoding:
+                self._warn_unknown_encoding(response, content_encoding)
+            response.headers["Content-Encoding"] = content_encoding
+            if self.stats:
+                self.stats.inc_value(
+                    "httpcompression/response_bytes",
+                    len(decoded_body),
+                )
+                self.stats.inc_value("httpcompression/response_count")
+            respcls = responsetypes.from_args(
+                headers=response.headers, url=response.url, body=decoded_body
+            )
+            kwargs: dict[str, Any] = {"body": decoded_body}
+            if issubclass(respcls, TextResponse):
+                # force recalculating the encoding until we make sure the
+                # responsetypes guessing is reliable
+                kwargs["encoding"] = None
+            response = response.replace(cls=respcls, **kwargs)
+            if not content_encoding:
+                del response.headers["Content-Encoding"]
         return response
 
     def _handle_encoding(

scrapy/downloadermiddlewares/offsite.py

@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import logging
 import re
-import warnings
 from typing import TYPE_CHECKING
 
 from scrapy import Request, Spider, signals
@@ -75,25 +74,10 @@ class OffsiteMiddleware:
         allowed_domains = getattr(spider, "allowed_domains", None)
         if not allowed_domains:
             return re.compile("")  # allow all by default
-        url_pattern = re.compile(r"^https?://.*$")
-        port_pattern = re.compile(r":\d+$")
         domains = []
         for domain in allowed_domains:
             if domain is None:
                 continue
-            if url_pattern.match(domain):
-                message = (
-                    "allowed_domains accepts only domains, not URLs. "
-                    f"Ignoring URL entry {domain} in allowed_domains."
-                )
-                warnings.warn(message, stacklevel=2)
-            elif port_pattern.search(domain):
-                message = (
-                    "allowed_domains accepts only domains without ports. "
-                    f"Ignoring entry {domain} in allowed_domains."
-                )
-                warnings.warn(message, stacklevel=2)
-            else:
-                domains.append(re.escape(domain))
+            domains.append(re.escape(domain))
         regex = rf"^(.*\.)?({'|'.join(domains)})$"
         return re.compile(regex)

scrapy/downloadermiddlewares/redirect.py

@@ -196,10 +196,7 @@ class BaseRedirectMiddleware:
 
 
 class RedirectMiddleware(BaseRedirectMiddleware):
-    """
-    Handle redirection of requests based on response status
-    and meta-refresh html tag.
-    """
+    """Handle redirection of requests based on response status."""
 
     @_warn_spider_arg
     def process_response(
@@ -251,6 +248,8 @@ class RedirectMiddleware(BaseRedirectMiddleware):
 
 
 class MetaRefreshMiddleware(BaseRedirectMiddleware):
+    """Handle redirection of requests based on meta-refresh html tag."""
+
     enabled_setting = "METAREFRESH_ENABLED"
 
     def __init__(self, settings: BaseSettings):

scrapy/downloadermiddlewares/retry.py

@@ -5,9 +5,6 @@ problems such as a connection timeout or HTTP 500 error.
 You can change the behaviour of this middleware by modifying the scraping settings:
 RETRY_TIMES - how many times to retry a failed page
 RETRY_HTTP_CODES - which HTTP response codes to retry
-
-Failed pages are collected on the scraping process and rescheduled at the end,
-once the spider has finished crawling all regular (non-failed) pages.
 """
 
 from __future__ import annotations
@@ -70,8 +67,9 @@ def get_retry_request(
     and :ref:`stats <topics-stats>`, and to provide extra logging context (see
     :func:`logging.debug`).
 
-    *reason* is a string or an :class:`Exception` object that indicates the
-    reason why the request needs to be retried. It is used to name retry stats.
+    *reason* is a string, an :class:`Exception` subclass or an
+    :class:`Exception` object that indicates the reason why the request needs
+    to be retried. It is used to name retry stats.
 
     *max_retry_times* is a number that determines the maximum number of times
     that *request* can be retried. If not specified or ``None``, the number is

scrapy/exceptions.py

@@ -115,7 +115,7 @@ class UsageError(Exception):
 
 class ScrapyDeprecationWarning(Warning):
     """Warning category for deprecated features, since the default
-    DeprecationWarning is silenced on Python 2.7+
+    :exc:`DeprecationWarning` is silenced.
     """
 
 

scrapy/exporters.py

@@ -171,7 +171,15 @@ class XmlItemExporter(BaseItemExporter):
         super().__init__(**kwargs)
         if not self.encoding:
             self.encoding = "utf-8"
-        self.xg = XMLGenerator(file, encoding=self.encoding)
+        # copied from xml.sax.saxutils._gettextwriter()
+        self.stream = TextIOWrapper(
+            file,
+            encoding=self.encoding,
+            errors="xmlcharrefreplace",
+            newline="\n",
+            write_through=True,
+        )
+        self.xg = XMLGenerator(self.stream, encoding=self.encoding)
 
     def _beautify_newline(self, new_item: bool = False) -> None:
         if self.indent is not None and (self.indent > 0 or new_item):
@@ -199,6 +207,7 @@ class XmlItemExporter(BaseItemExporter):
     def finish_exporting(self) -> None:
         self.xg.endElement(self.root_element)
         self.xg.endDocument()
+        self.stream.detach()  # Avoid closing the wrapped file.
 
     def _export_xml_field(self, name: str, serialized_value: Any, depth: int) -> None:
         self._beautify_indent(depth=depth)

scrapy/extensions/feedexport.py

@@ -250,20 +250,22 @@ class S3FeedStorage(BlockingFeedStorage):
 
     def _store_in_thread(self, file: IO[bytes]) -> None:
         file.seek(0)
-        if self.acl:
-            self.s3_client.upload_fileobj(
-                Bucket=self.bucketname,
-                Key=self.keyname,
-                Fileobj=file,
-                ExtraArgs={"ACL": self.acl},
-            )
-        else:
-            self.s3_client.upload_fileobj(
-                Bucket=self.bucketname,
-                Key=self.keyname,
-                Fileobj=file,
-            )
-        file.close()
+        try:
+            if self.acl:
+                self.s3_client.upload_fileobj(
+                    Bucket=self.bucketname,
+                    Key=self.keyname,
+                    Fileobj=file,
+                    ExtraArgs={"ACL": self.acl},
+                )
+            else:
+                self.s3_client.upload_fileobj(
+                    Bucket=self.bucketname,
+                    Key=self.keyname,
+                    Fileobj=file,
+                )
+        finally:
+            file.close()
 
 
 class GCSFeedStorage(BlockingFeedStorage):

scrapy/extensions/logstats.py

@@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
 
 class LogStats:
     """Log basic scraping stats periodically like:
-    * RPM - Requests per Minute
+    * RPM - Responses per Minute
     * IPM - Items per Minute
     """
 

scrapy/extensions/memusage.py

@@ -113,7 +113,7 @@ class MemoryUsage:
                 {"memusage": mem},
                 extra={"crawler": self.crawler},
             )
-            if self.notify_mails:
+            if self.notify_mails:  # pragma: no cover
                 subj = (
                     f"{self.crawler.settings['BOT_NAME']} terminated: "
                     f"memory usage exceeded {mem}MiB at {socket.gethostname()}"
@@ -146,7 +146,7 @@ class MemoryUsage:
                 {"memusage": mem},
                 extra={"crawler": self.crawler},
             )
-            if self.notify_mails:
+            if self.notify_mails:  # pragma: no cover
                 subj = (
                     f"{self.crawler.settings['BOT_NAME']} warning: "
                     f"memory usage reached {mem}MiB at {socket.gethostname()}"
@@ -155,7 +155,7 @@ class MemoryUsage:
                 self.crawler.stats.set_value("memusage/warning_notified", 1)
             self.warned = True
 
-    def _send_report(self, rcpts: list[str], subject: str) -> None:
+    def _send_report(self, rcpts: list[str], subject: str) -> None:  # pragma: no cover
         """send notification mail with some additional useful info"""
         assert self.crawler.engine
         assert self.crawler.stats

scrapy/extensions/throttle.py

@@ -116,7 +116,7 @@ class AutoThrottle:
         # It works better with problematic sites.
         new_delay = max(target_delay, new_delay)
 
-        # Make sure self.mindelay <= new_delay <= self.max_delay
+        # Make sure self.mindelay <= new_delay <= self.maxdelay
         new_delay = min(max(self.mindelay, new_delay), self.maxdelay)
 
         # Dont adjust delay if response status != 200 and new delay is smaller

scrapy/http/__init__.py

@@ -5,11 +5,9 @@ Use this module (instead of the more specific ones) when importing Headers,
 Request and Response outside this module.
 """
 
-from warnings import catch_warnings, filterwarnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http.headers import Headers
 from scrapy.http.request import Request
+from scrapy.http.request.form import FormRequest
 from scrapy.http.request.json_request import JsonRequest
 from scrapy.http.request.rpc import XmlRpcRequest
 from scrapy.http.response import Response
@@ -17,19 +15,6 @@ from scrapy.http.response.html import HtmlResponse
 from scrapy.http.response.json import JsonResponse
 from scrapy.http.response.text import TextResponse
 from scrapy.http.response.xml import XmlResponse
-from scrapy.utils.deprecate import create_deprecated_class
-
-with catch_warnings():
-    filterwarnings("ignore", category=ScrapyDeprecationWarning)
-
-    from scrapy.http.request.form import FormRequest as _FormRequest
-
-    FormRequest = create_deprecated_class(
-        name="FormRequest",
-        new_class=_FormRequest,
-        subclass_warn_message="{cls} inherits from deprecated class {old}, use the form2request library instead.",
-        instance_warn_message="{cls} is deprecated, use the form2request library instead.",
-    )
 
 __all__ = [
     "FormRequest",

scrapy/http/cookies.py

@@ -136,9 +136,9 @@ class _DummyLock:
 
 
 class WrappedRequest:
-    """Wraps a scrapy Request class with methods defined by urllib2.Request class to interact with CookieJar class
-
-    see http://docs.python.org/library/urllib2.html#urllib2.Request
+    """Wraps a :class:`scrapy.Request` class with methods defined by
+    the :class:`urllib.request.Request` class to interact with
+    the :class:`http.cookiejar.CookieJar` class.
     """
 
     def __init__(self, request: Request):

scrapy/http/request/__init__.py

@@ -51,7 +51,7 @@ class VerboseCookie(TypedDict):
     secure: NotRequired[bool]
 
 
-CookiesT: TypeAlias = dict[str, str] | list[VerboseCookie]
+CookiesT: TypeAlias = dict[str | bytes, str | bytes] | list[VerboseCookie]
 
 
 RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")

scrapy/http/request/form.py

@@ -32,19 +32,61 @@ if TYPE_CHECKING:
 
     from scrapy.http.response.text import TextResponse
 
-warn(
-    "The entire scrapy.http.request.form module is deprecated. Use the "
-    "form2request library instead.",
-    ScrapyDeprecationWarning,
-    stacklevel=2,
-)
-
 FormdataVType: TypeAlias = str | Iterable[str]
 FormdataKVType: TypeAlias = tuple[str, FormdataVType]
 FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None
 
 
 class FormRequest(Request):
+    """A :class:`~scrapy.Request` subclass with a ``formdata`` parameter that
+    url-encodes the given data and assigns it to the request, which makes it
+    convenient to send arbitrary form data via HTTP POST or GET without an HTML
+    ``<form>`` element to parse.
+
+    .. note:: To build a request from an HTML ``<form>`` element found in a
+       response, use :doc:`form2request <form2request:index>` instead. See
+       :ref:`form`.
+
+    The remaining arguments are the same as for the :class:`~scrapy.Request`
+    class and are not documented here.
+
+    :param formdata: a dictionary (or iterable of (key, value) tuples)
+       containing HTML form data which will be url-encoded. If
+       :attr:`~scrapy.Request.method` is not given and ``formdata`` is
+       provided, the method is set to ``"POST"`` and the data is assigned to
+       the request body; if the method is ``"GET"``, the data is added to the
+       URL query string instead.
+    :type formdata: dict or collections.abc.Iterable
+
+    To send data via HTTP POST, simulating an HTML form submission, return a
+    :class:`~scrapy.FormRequest` object from your spider:
+
+    .. skip: next
+    .. code-block:: python
+
+        return [
+            FormRequest(
+                url="http://www.example.com/post/action",
+                formdata={"name": "John Doe", "age": "27"},
+                callback=self.after_post,
+            )
+        ]
+
+    To send the data in the URL query string instead, use the ``GET`` method:
+
+    .. skip: next
+    .. code-block:: python
+
+        return [
+            FormRequest(
+                url="http://www.example.com/search",
+                method="GET",
+                formdata={"q": "keyword", "page": "1"},
+                callback=self.parse_results,
+            )
+        ]
+    """
+
     __slots__ = ()
 
     valid_form_methods: ClassVar[list[str]] = ["GET", "POST"]
@@ -84,6 +126,13 @@ class FormRequest(Request):
         formcss: str | None = None,
         **kwargs: Any,
     ) -> Self:
+        warn(
+            "FormRequest.from_response() is deprecated. Use the form2request "
+            "library instead.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+
         kwargs.setdefault("encoding", response.encoding)
 
         if formcss is not None:

scrapy/http/response/html.py

@@ -1,6 +1,7 @@
 """
-This module implements the HtmlResponse class which adds encoding
-discovering through HTML encoding declarations to the TextResponse class.
+This module implements the :class:`HtmlResponse` class which is used as a
+content type marker by :class:`~scrapy.selector.Selector` and can be used in
+``isinstance()`` checks.
 
 See documentation in docs/topics/request-response.rst
 """

scrapy/http/response/xml.py

@@ -1,6 +1,7 @@
 """
-This module implements the XmlResponse class which adds encoding
-discovering through XML encoding declarations to the TextResponse class.
+This module implements the :class:`XmlResponse` class which is used as a
+content type marker by :class:`~scrapy.selector.Selector` and can be used in
+``isinstance()`` checks.
 
 See documentation in docs/topics/request-response.rst
 """

scrapy/item.py

@@ -1,7 +1,7 @@
 """
 Scrapy Item
 
-See documentation in docs/topics/item.rst
+See documentation in docs/topics/items.rst
 """
 
 from __future__ import annotations
@@ -25,6 +25,25 @@ class Field(dict[str, Any]):
     """Container of field metadata"""
 
 
+def _ordered_field_names(cls: type) -> list[str]:
+    """Return the names of the :class:`Field` attributes of *cls* in definition
+    order.
+
+    Fields declared in base classes come first, ordered from the topmost base
+    to the most derived class. Within each class, fields keep their definition
+    order. A field redefined in a subclass keeps the position of its first
+    definition.
+    """
+    names: list[str] = []
+    seen: set[str] = set()
+    for base in reversed(cls.__mro__):
+        for name, value in vars(base).items():
+            if isinstance(value, Field) and name not in seen:
+                seen.add(name)
+                names.append(name)
+    return names
+
+
 class ItemMeta(ABCMeta):
     """Metaclass_ of :class:`Item` that handles field definitions.
 
@@ -39,13 +58,9 @@ class ItemMeta(ABCMeta):
         _class = super().__new__(mcs, "x_" + class_name, new_bases, attrs)
 
         fields = getattr(_class, "fields", {})
-        new_attrs = {}
-        for n in dir(_class):
-            v = getattr(_class, n)
-            if isinstance(v, Field):
-                fields[n] = v
-            elif n in attrs:
-                new_attrs[n] = attrs[n]
+        for n in _ordered_field_names(_class):
+            fields[n] = getattr(_class, n)
+        new_attrs = {n: v for n, v in attrs.items() if not isinstance(v, Field)}
 
         new_attrs["fields"] = fields
         new_attrs["_class"] = _class
@@ -80,6 +95,14 @@ class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta):
     #: those populated. The keys are the field names and the values are the
     #: :class:`Field` objects used in the :ref:`Item declaration
     #: <topics-items-declaring>`.
+    #:
+    #: Fields are kept in definition order: fields declared in base classes
+    #: come first, followed by fields declared in subclasses, and a field
+    #: redefined in a subclass keeps the position of its first definition.
+    #:
+    #: .. versionchanged:: VERSION
+    #:    Fields are now returned in definition order rather than alphabetical
+    #:    order.
     fields: dict[str, Field]
 
     def __init__(self, *args: Any, **kwargs: Any):

scrapy/link.py

@@ -9,7 +9,9 @@ its documentation in: docs/topics/link-extractors.rst
 class Link:
     """Link objects represent an extracted link by the LinkExtractor.
 
-    Using the anchor tag sample below to illustrate the parameters::
+    Using the anchor tag sample below to illustrate the parameters:
+
+    .. code-block:: html
 
             <a href="https://example.com/nofollow.html#foo" rel="nofollow">Dont follow this one</a>
 
@@ -39,7 +41,7 @@ class Link:
 
     def __eq__(self, other: object) -> bool:
         if not isinstance(other, Link):
-            raise NotImplementedError
+            return NotImplemented
         return (
             self.url == other.url
             and self.text == other.text

scrapy/loader/__init__.py

@@ -27,7 +27,7 @@ class ItemLoader(itemloaders.ItemLoader):
     :param item: The item instance to populate using subsequent calls to
         :meth:`~ItemLoader.add_xpath`, :meth:`~ItemLoader.add_css`,
         or :meth:`~ItemLoader.add_value`.
-    :type item: scrapy.item.Item
+    :type item: :ref:`item object <item-types>`
 
     :param selector: The selector to extract data from, when using the
         :meth:`add_xpath`, :meth:`add_css`, :meth:`replace_xpath`, or

scrapy/logformatter.py

@@ -58,18 +58,20 @@ class LogFormatter:
     logging an action the method must return ``None``.
 
     Here is an example on how to create a custom log formatter to lower the severity level of
-    the log message when an item is dropped from the pipeline::
+    the log message when an item is dropped from the pipeline:
 
-            class PoliteLogFormatter(logformatter.LogFormatter):
-                def dropped(self, item, exception, response, spider):
-                    return {
-                        'level': logging.INFO, # lowering the level from logging.WARNING
-                        'msg': "Dropped: %(exception)s" + os.linesep + "%(item)s",
-                        'args': {
-                            'exception': exception,
-                            'item': item,
-                        }
-                    }
+    .. code-block:: python
+
+        class PoliteLogFormatter(logformatter.LogFormatter):
+            def dropped(self, item, exception, response, spider):
+                return {
+                    "level": logging.INFO,  # lowering the level from logging.WARNING
+                    "msg": "Dropped: %(exception)s" + os.linesep + "%(item)s",
+                    "args": {
+                        "exception": exception,
+                        "item": item,
+                    },
+                }
     """
 
     def crawled(

scrapy/mail.py

@@ -1,7 +1,5 @@
 """
 Mail sending helpers
-
-See documentation in docs/topics/email.rst
 """
 
 from __future__ import annotations

scrapy/pipelines/__init__.py

@@ -1,7 +1,7 @@
 """
 Item pipeline
 
-See documentation in docs/item-pipeline.rst
+See documentation in docs/topics/item-pipeline.rst
 """
 
 from __future__ import annotations

scrapy/pipelines/files.py

@@ -423,7 +423,7 @@ class FTPFilesStore:
 
 
 class FilesPipeline(MediaPipeline):
-    """Abstract pipeline that implement the file downloading
+    """Pipeline that implements file downloading.
 
     This pipeline tries to minimize network transfers and file processing,
     doing stat of the files and determining if file is new, up-to-date or

scrapy/pipelines/images.py

@@ -11,14 +11,20 @@ import hashlib
 import warnings
 from contextlib import suppress
 from io import BytesIO
-from typing import TYPE_CHECKING, Any, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar, cast
 
 from itemadapter import ItemAdapter
 
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
-from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum
+from scrapy.pipelines.files import (
+    FileException,
+    FilesPipeline,
+    GCSFilesStore,
+    S3FilesStore,
+    _md5sum,
+)
 from scrapy.utils.defer import ensure_awaitable
 from scrapy.utils.python import to_bytes
 
@@ -33,6 +39,7 @@ if TYPE_CHECKING:
 
     from scrapy.crawler import Crawler
     from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
+    from scrapy.settings import BaseSettings
 
 
 class ImageException(FileException):
@@ -40,7 +47,7 @@ class ImageException(FileException):
 
 
 class ImagesPipeline(FilesPipeline):
-    """Abstract pipeline that implement the image thumbnail generation logic"""
+    """Pipeline that implements the handling logic specific to images."""
 
     MEDIA_NAME: str = "image"
 
@@ -126,6 +133,20 @@ class ImagesPipeline(FilesPipeline):
     ) -> str:
         return await self.image_downloaded(response, request, info, item=item)
 
+    @classmethod
+    def _update_stores(cls, settings: BaseSettings) -> None:
+        super()._update_stores(settings)
+
+        s3store: type[S3FilesStore] = cast(
+            "type[S3FilesStore]", cls.STORE_SCHEMES["s3"]
+        )
+        s3store.POLICY = settings["IMAGES_STORE_S3_ACL"]
+
+        gcs_store: type[GCSFilesStore] = cast(
+            "type[GCSFilesStore]", cls.STORE_SCHEMES["gs"]
+        )
+        gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None
+
     async def image_downloaded(
         self,
         response: Response,

scrapy/pqueues.py

@@ -92,9 +92,10 @@ class ScrapyPriorityQueue:
     -   The :data:`~scrapy.Request.priority` of the request.
 
     For each combination of the above seen, this class creates an instance of
-    *downstream_queue_cls* with *key* set to a subdirectory of the persistence
-    directory, named as the request priority (e.g. ``1``), with an ``s`` suffix
-    in case of a start request (e.g. ``1s``).
+    *downstream_queue_cls* (or *start_queue_cls* for start requests if it was
+    passed) with *key* set to a subdirectory of the persistence directory,
+    named as the negated request priority (e.g. ``-1``), with an ``s`` suffix
+    in case of a start request (e.g. ``-1s``).
     """
 
     @classmethod
@@ -141,10 +142,14 @@ class ScrapyPriorityQueue:
             q = self.qfactory(priority)
             if q:
                 self.queues[priority] = q
+            else:
+                q.close()
             if self._start_queue_cls:
                 q = self._sqfactory(priority)
                 if q:
                     self._start_queues[priority] = q
+                else:
+                    q.close()
 
         self.curprio = min(startprios)
 

scrapy/resolver.py

@@ -75,7 +75,7 @@ class HostResolution:
     def __init__(self, name: str):
         self.name: str = name
 
-    def cancel(self) -> None:
+    def cancel(self) -> None:  # pragma: no cover
         raise NotImplementedError
 
 

scrapy/responsetypes.py

@@ -40,18 +40,13 @@ class ResponseTypes:
         self.classes: dict[str, type[Response]] = {}
         self.mimetypes: MimeTypes = MimeTypes()
         mimedata = get_data("scrapy", "mime.types")
-        if not mimedata:
-            raise ValueError(
-                "The mime.types file is not found in the Scrapy installation"
-            )
+        assert mimedata is not None
         self.mimetypes.readfp(StringIO(mimedata.decode("utf8")))
         for mimetype, cls in self.CLASSES.items():
             self.classes[mimetype] = load_object(cls)
 
     def from_mimetype(self, mimetype: str) -> type[Response]:
         """Return the most appropriate Response class for the given mimetype"""
-        if mimetype is None:
-            return Response
         if mimetype in self.classes:
             return self.classes[mimetype]
         basetype = f"{mimetype.split('/', maxsplit=1)[0]}/*"

scrapy/selector/unified.py

@@ -1,10 +1,6 @@
-"""
-XPath selectors based on lxml
-"""
-
 from __future__ import annotations
 
-from typing import Any
+from typing import Any, Literal
 
 from parsel import Selector as _ParselSelector
 
@@ -18,13 +14,10 @@ __all__ = ["Selector", "SelectorList"]
 _NOT_SET = object()
 
 
-def _st(response: TextResponse | None, st: str | None) -> str:
-    if st is None:
-        return "xml" if isinstance(response, XmlResponse) else "html"
-    return st
+SelectorType = Literal["html", "xml", "json", "text"]
 
 
-def _response_from_text(text: str | bytes, st: str | None) -> TextResponse:
+def _response_from_text(text: str | bytes, st: SelectorType | None) -> TextResponse:
     rt: type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse
     return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8"))
 
@@ -49,23 +42,16 @@ class Selector(_ParselSelector, object_ref):
     ``response`` isn't available. Using ``text`` and ``response`` together is
     undefined behavior.
 
-    ``type`` defines the selector type, it can be ``"html"``, ``"xml"``, ``"json"``
-    or ``None`` (default).
+    ``type`` defines the selector type, it can be ``"html"``, ``"xml"``,
+    ``"json"``, ``"text"`` or ``None`` (default). It's passed to
+    :class:`parsel.Selector` and its meaning is defined there. However, when
+    ``type`` is ``None``, it is set to ``"xml"`` for an
+    :class:`~scrapy.http.XmlResponse` and to ``"html"`` otherwise before
+    passing it to :class:`parsel.Selector`.
 
-    If ``type`` is ``None``, the selector automatically chooses the best type
-    based on ``response`` type (see below), or defaults to ``"html"`` in case it
-    is used together with ``text``.
-
-    If ``type`` is ``None`` and a ``response`` is passed, the selector type is
-    inferred from the response type as follows:
-
-    * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
-    * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
-    * ``"json"`` for :class:`~scrapy.http.TextResponse` type
-    * ``"html"`` for anything else
-
-    Otherwise, if ``type`` is set, the selector type will be forced and no
-    detection will occur.
+    .. note:: JSON selector support requires ``parsel`` 1.8.0 or higher. With
+       older versions setting ``type`` to ``"json"`` or ``"text"`` is not
+       supported.
     """
 
     __slots__ = ["response"]
@@ -75,7 +61,7 @@ class Selector(_ParselSelector, object_ref):
         self,
         response: TextResponse | None = None,
         text: str | None = None,
-        type: str | None = None,  # noqa: A002
+        type: SelectorType | None = None,  # noqa: A002
         root: Any | None = _NOT_SET,
         **kwargs: Any,
     ):
@@ -84,10 +70,11 @@ class Selector(_ParselSelector, object_ref):
                 f"{self.__class__.__name__}.__init__() received both response and text"
             )
 
-        st = _st(response, type)
+        if type is None:
+            type = "xml" if isinstance(response, XmlResponse) else "html"  # noqa: A001
 
         if text is not None:
-            response = _response_from_text(text, st)
+            response = _response_from_text(text, type)
 
         if response is not None:
             text = response.text
@@ -98,4 +85,4 @@ class Selector(_ParselSelector, object_ref):
         if root is not _NOT_SET:
             kwargs["root"] = root
 
-        super().__init__(text=text, type=st, **kwargs)
+        super().__init__(text=text, type=type, **kwargs)

scrapy/settings/__init__.py

@@ -454,9 +454,10 @@ class BaseSettings(MutableMapping[str, Any]):
         """
         Store a key/value attribute with a given priority.
 
-        Settings should be populated *before* configuring the Crawler object
-        (through the :meth:`~scrapy.crawler.Crawler.configure` method),
-        otherwise they won't have any effect.
+        Settings should be populated *before* the Crawler object applies them
+        (in the :meth:`~scrapy.crawler.Crawler.crawl_async` or
+        :meth:`~scrapy.crawler.Crawler.crawl` method), otherwise they won't
+        have any effect.
 
         :param name: the setting name
         :type name: str
@@ -613,7 +614,7 @@ class BaseSettings(MutableMapping[str, Any]):
         """
         Make a deep copy of current settings.
 
-        This method returns a new instance of the :class:`Settings` class,
+        This method returns a new instance of this class,
         populated with the same values and their priorities.
 
         Modifications to the new object won't be reflected on the original
@@ -658,7 +659,7 @@ class BaseSettings(MutableMapping[str, Any]):
         Make a copy of current settings and convert to a dict.
 
         This method returns a new dict populated with the same values
-        and their priorities as the current settings.
+        as the current settings.
 
         Modifications to the returned dict won't be reflected on the original
         settings.

scrapy/settings/default_settings.py

@@ -107,6 +107,9 @@ __all__ = [
     "FTP_PASSWORD",
     "FTP_USER",
     "GCS_PROJECT_ID",
+    "HTTPAUTH_DOMAIN",
+    "HTTPAUTH_PASS",
+    "HTTPAUTH_USER",
     "HTTPCACHE_ALWAYS_STORE",
     "HTTPCACHE_DBM_MODULE",
     "HTTPCACHE_DIR",
@@ -397,6 +400,10 @@ FTP_PASSWORD = "guest"  # noqa: S105
 
 GCS_PROJECT_ID = None
 
+HTTPAUTH_USER = ""
+HTTPAUTH_PASS = ""
+HTTPAUTH_DOMAIN = None
+
 HTTPCACHE_ENABLED = False
 HTTPCACHE_ALWAYS_STORE = False
 HTTPCACHE_DBM_MODULE = "dbm"

scrapy/shell.py

@@ -30,7 +30,6 @@ from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
 from scrapy.utils.misc import load_object
-from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop
 from scrapy.utils.response import open_in_browser
 
 if TYPE_CHECKING:
@@ -76,8 +75,8 @@ if TYPE_CHECKING:
 # running event loop.
 #
 # Side note: it should be possible to remove _request_deferred() by using
-# engine.download() instead of engine.schedule(), losing the usual stuff like
-# spider middlewares (none of which should be important).
+# engine.download_async() instead of engine.crawl(), losing the usual stuff
+# like spider middlewares (none of which should be important).
 #
 # Other architecture problems:
 # * scrapy.cmdline.execute() creates an AsyncCrawlerProcess instance which
@@ -189,12 +188,9 @@ class Shell:
     async def _schedule(self, request: Request, spider: Spider | None) -> Response:
         """Send the request to the engine, wait for the result.
 
-        Runs in the reactor thread.
+        Runs in the reactor thread when using the reactor, or in the asyncio
+        event loop thread otherwise.
         """
-        if self._use_reactor and is_asyncio_reactor_installed():
-            # set the asyncio event loop for the current thread
-            event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"]
-            set_asyncio_event_loop(event_loop_path)
         if not self.spider:
             await self._open_spider(spider)
         assert self.crawler.engine is not None

scrapy/spidermiddlewares/base.py

@@ -75,7 +75,7 @@ class BaseSpiderMiddleware:
     ) -> Request | None:
         """Return a processed request from the spider output.
 
-        This method is called with a single request from the start seeds or the
+        This method is called with a single request from ``start()`` or the
         spider output. It should return the same or a different request, or
         ``None`` to ignore it.
 
@@ -84,7 +84,7 @@ class BaseSpiderMiddleware:
 
         :param response: the response being processed
         :type response: :class:`~scrapy.http.Response` object or ``None`` for
-            start seeds
+            start requests
 
         :return: the processed request or ``None``
         """
@@ -93,7 +93,7 @@ class BaseSpiderMiddleware:
     def get_processed_item(self, item: Any, response: Response | None) -> Any:
         """Return a processed item from the spider output.
 
-        This method is called with a single item from the start seeds or the
+        This method is called with a single item from ``start()`` or the
         spider output. It should return the same or a different item, or
         ``None`` to ignore it.
 
@@ -102,7 +102,7 @@ class BaseSpiderMiddleware:
 
         :param response: the response being processed
         :type response: :class:`~scrapy.http.Response` object or ``None`` for
-            start seeds
+            start items
 
         :return: the processed item or ``None``
         """

scrapy/spidermiddlewares/httperror.py

@@ -28,7 +28,7 @@ logger = logging.getLogger(__name__)
 
 
 class HttpError(IgnoreRequest):
-    """A non-200 response was filtered"""
+    """A non-2xx response was filtered"""
 
     def __init__(self, response: Response, *args: Any, **kwargs: Any):
         self.response = response

scrapy/spidermiddlewares/referer.py

@@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, cast
 from urllib.parse import urlparse
 from warnings import warn
 
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
 from scrapy.utils.misc import load_object
@@ -92,7 +92,7 @@ class ReferrerPolicy(ABC):
         )
 
     def origin(self, url: str) -> str | None:
-        """Return serialized origin (scheme, host, path) for a request or response URL."""
+        """Return serialized origin (scheme, host, port) for a request or response URL."""
         return self.strip_url(url, origin_only=True)
 
     def potentially_trustworthy(self, url: str) -> bool:
@@ -308,6 +308,12 @@ class RefererMiddleware(BaseSpiderMiddleware):
         # Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
         self.policies[""] = NoReferrerWhenDowngradePolicy
         if settings is None:
+            warn(
+                "Instantiating RefererMiddleware without a 'settings' argument is "
+                "deprecated.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
             return
         setting_policies = settings.getdict("REFERRER_POLICIES")
         for policy_name, policy_class_import_path in setting_policies.items():
@@ -349,7 +355,7 @@ class RefererMiddleware(BaseSpiderMiddleware):
             response = kwargs.pop("resp_or_url")
             warn(
                 "Passing 'resp_or_url' is deprecated, use 'response' instead.",
-                DeprecationWarning,
+                ScrapyDeprecationWarning,
                 stacklevel=2,
             )
         if response is None:
@@ -360,7 +366,7 @@ class RefererMiddleware(BaseSpiderMiddleware):
             warn(
                 "Passing a response URL to RefererMiddleware.policy() instead "
                 "of a Response object is deprecated.",
-                DeprecationWarning,
+                ScrapyDeprecationWarning,
                 stacklevel=2,
             )
         allow_import_path = True

scrapy/spiders/__init__.py

@@ -125,12 +125,6 @@ class Spider(object_ref):
 
         .. seealso:: :ref:`start-requests`
         """
-        if not self.start_urls and hasattr(self, "start_url"):
-            raise AttributeError(
-                "Crawling could not start: 'start_urls' not found "
-                "or empty (but found 'start_url' attribute instead, "
-                "did you miss an 's'?)"
-            )
         for url in self.start_urls:
             yield Request(url, dont_filter=True)
 

scrapy/spiders/crawl.py

@@ -110,6 +110,7 @@ class CrawlSpider(Spider):
                 "deprecated: it will be removed in future Scrapy releases. "
                 "Please override the CrawlSpider.parse_with_rules method "
                 "instead.",
+                ScrapyDeprecationWarning,
                 stacklevel=2,
             )
 
@@ -199,6 +200,7 @@ class CrawlSpider(Spider):
             "The CrawlSpider._parse_response method is deprecated: "
             "it will be removed in future Scrapy releases. "
             "Please use the CrawlSpider.parse_with_rules method instead.",
+            ScrapyDeprecationWarning,
             stacklevel=2,
         )
         return self.parse_with_rules(response, callback, cb_kwargs, follow)

scrapy/spiders/feed.py

@@ -54,17 +54,21 @@ class XMLFeedSpider(Spider):
         return response
 
     def parse_node(self, response: Response, selector: Selector) -> Any:
-        """This method must be overridden with your custom spider functionality"""
+        """This method is called for the nodes matching the provided tag name
+        (itertag). Receives the response and an Selector for each node.
+
+        This method must return either an item, a request, or a list
+        containing any of them.
+
+        This method must be overridden with your custom spider functionality.
+        """
         if hasattr(self, "parse_item"):  # backward compatibility
             return self.parse_item(response, selector)
         raise NotImplementedError
 
     def parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any:
         """This method is called for the nodes matching the provided tag name
-        (itertag). Receives the response and an Selector for each node.
-        Overriding this method is mandatory. Otherwise, you spider won't work.
-        This method must return either an item, a request, or a list
-        containing any of them.
+        (itertag). Receives the response and an iterable of Selectors.
         """
 
         for selector in nodes:
@@ -113,6 +117,9 @@ class CSVFeedSpider(Spider):
     It receives a CSV file in a response; iterates through each of its rows,
     and calls parse_row with a dict containing each field's data.
 
+    This spider also gives the opportunity to override adapt_response and
+    process_results methods for pre and post-processing purposes.
+
     You can set some options regarding the CSV file, such as the delimiter, quotechar
     and the file's headers.
     """
@@ -136,16 +143,14 @@ class CSVFeedSpider(Spider):
         return response
 
     def parse_row(self, response: Response, row: dict[str, str]) -> Any:
-        """This method must be overridden with your custom spider functionality"""
+        """Receives a response and a dict (representing each row) with a key for
+        each provided (or detected) header of the CSV file.
+
+        This method must be overridden with your custom spider functionality.
+        """
         raise NotImplementedError
 
     def parse_rows(self, response: Response) -> Any:
-        """Receives a response and a dict (representing each row) with a key for
-        each provided (or detected) header of the CSV file.  This spider also
-        gives the opportunity to override adapt_response and
-        process_results methods for pre and post-processing purposes.
-        """
-
         for row in csviter(
             response, self.delimiter, self.headers, quotechar=self.quotechar
         ):

scrapy/templates/project/module/pipelines.py.tmpl

@@ -9,5 +9,5 @@ from itemadapter import ItemAdapter
 
 
 class ${ProjectName}Pipeline:
-    def process_item(self, item, spider):
+    def process_item(self, item):
         return item

scrapy/utils/_deps_compat.py

@@ -1,7 +1,15 @@
+import sys
+
 from OpenSSL import __version__ as PYOPENSSL_VERSION_STRING
 from packaging.version import Version
 from twisted import version as TWISTED_VERSION
 from twisted.python.versions import Version as TxVersion
+from w3lib import __version__ as W3LIB_VERSION_STRING
+
+# improved urllib.robotparser, https://github.com/python/cpython/pull/149374
+STDLIB_IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5) or (
+    (3, 13, 14) <= sys.version_info < (3, 14)
+)
 
 TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0)
 # changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506
@@ -10,7 +18,11 @@ TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
 TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0)
 
 PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)
-# SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object
-PYOPENSSL_WANTS_X509_PKEY = PYOPENSSL_VERSION < Version("24.3.0")
+# pyOpenSSL X.509 APIs are deprecated and cryptography-based ones are preferred
+PYOPENSSL_X509_DEPRECATED = PYOPENSSL_VERSION >= Version("24.3.0")
 # SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable
 PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0")
+
+W3LIB_VERSION = Version(W3LIB_VERSION_STRING)
+# safe_url_string() strips the input, https://github.com/scrapy/w3lib/pull/207
+W3LIB_STRIPS_URLS = W3LIB_VERSION >= Version("2.1.1")

scrapy/utils/datatypes.py

@@ -1,8 +1,6 @@
 """
 This module contains data types used by Scrapy which are not included in the
 Python Standard Library.
-
-This module must not depend on any module outside the Standard Library.
 """
 
 from __future__ import annotations
@@ -152,7 +150,9 @@ class LocalCache(OrderedDict[_KT, _VT]):
         self.limit: int | None = limit
 
     def __setitem__(self, key: _KT, value: _VT) -> None:
-        if self.limit:
+        if self.limit is not None:
+            if self.limit == 0:
+                return
             while len(self) >= self.limit:
                 self.popitem(last=False)
         super().__setitem__(key, value)

scrapy/utils/defer.py

@@ -103,7 +103,7 @@ async def _defer_sleep_async() -> None:
 def defer_result(result: Any) -> Deferred[Any]:  # pragma: no cover
     warnings.warn(
         "scrapy.utils.defer.defer_result() is deprecated, use"
-        " twisted.internet.defer.success() and twisted.internet.defer.fail(),"
+        " twisted.internet.defer.succeed() and twisted.internet.defer.fail(),"
         " plus an explicit sleep if needed, or explicit reactor.callLater().",
         category=ScrapyDeprecationWarning,
         stacklevel=2,
@@ -469,22 +469,22 @@ def _maybeDeferred_coro(
 def deferred_to_future(d: Deferred[_T]) -> Future[_T]:
     """Return an :class:`asyncio.Future` object that wraps *d*.
 
-    This function requires
-    :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be
-    installed.
+    This function requires an installed asyncio reactor or a running asyncio
+    event loop, see :ref:`using-asyncio`.
 
-    When :ref:`using the asyncio reactor <install-asyncio>`, you cannot await
-    on :class:`~twisted.internet.defer.Deferred` objects from :ref:`Scrapy
-    callables defined as coroutines <coroutine-support>`, you can only await on
-    ``Future`` objects. Wrapping ``Deferred`` objects into ``Future`` objects
-    allows you to wait on them::
+    In this state you cannot await on :class:`~twisted.internet.defer.Deferred`
+    objects from :ref:`Scrapy callables defined as coroutines
+    <coroutine-support>`, you can only await on ``Future`` objects. Wrapping
+    ``Deferred`` objects into ``Future`` objects allows you to wait on them:
+
+    .. code-block:: python
 
         class MySpider(Spider):
             ...
+
             async def parse(self, response):
-                additional_request = scrapy.Request('https://example.org/price')
-                deferred = self.crawler.engine.download(additional_request)
-                additional_response = await deferred_to_future(deferred)
+                deferred = some_dfd_helper()
+                result = await deferred_to_future(deferred)
 
     .. versionchanged:: 2.14
         This function no longer installs an asyncio loop if called before the
@@ -492,7 +492,10 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]:
         in this case.
     """
     if not is_asyncio_available():
-        raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.")
+        raise RuntimeError(
+            "deferred_to_future() requires an installed asyncio reactor"
+            " or a running asyncio event loop."
+        )
     return d.asFuture(asyncio.get_event_loop())
 
 
@@ -501,23 +504,26 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]:
     defined as a coroutine <coroutine-support>`.
 
     What you can await in Scrapy callables defined as coroutines depends on the
-    value of :setting:`TWISTED_REACTOR`:
+    value of :setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED`:
 
-    -   When :ref:`using the asyncio reactor <install-asyncio>`, you can only
-        await on :class:`asyncio.Future` objects.
+    -   When :ref:`using the asyncio reactor <install-asyncio>`, or :ref:`not
+        using a reactor at all <asyncio-without-reactor>`, you can only await
+        on :class:`asyncio.Future` objects.
 
-    -   When not using the asyncio reactor, you can only await on
-        :class:`~twisted.internet.defer.Deferred` objects.
+    -   When :ref:`using a non-asyncio reactor <disable-asyncio>`, you can only
+        await on :class:`~twisted.internet.defer.Deferred` objects.
 
-    If you want to write code that uses ``Deferred`` objects but works with any
-    reactor, use this function on all ``Deferred`` objects::
+    If you want to write code that uses ``Deferred`` objects but works in both
+    of these states, use this function on all ``Deferred`` objects:
+
+    .. code-block:: python
 
         class MySpider(Spider):
             ...
+
             async def parse(self, response):
-                additional_request = scrapy.Request('https://example.org/price')
-                deferred = self.crawler.engine.download(additional_request)
-                additional_response = await maybe_deferred_to_future(deferred)
+                deferred = some_dfd_helper()
+                result = await maybe_deferred_to_future(deferred)
     """
     if not is_asyncio_available():
         return d

scrapy/utils/deprecate.py

@@ -43,15 +43,18 @@ def create_deprecated_class(
     It can be used to rename a base class in a library. For example, if we
     have
 
-        class OldName(SomeClass):
-            # ...
+    .. code-block:: python
 
-    and we want to rename it to NewName, we can do the following::
+        class OldName(SomeClass): ...
 
-        class NewName(SomeClass):
-            # ...
+    and we want to rename it to NewName, we can do the following:
 
-        OldName = create_deprecated_class('OldName', NewName)
+    .. code-block:: python
+
+        class NewName(SomeClass): ...
+
+
+        OldName = create_deprecated_class("OldName", NewName)
 
     Then, if user class inherits from OldName, warning is issued. Also, if
     some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)``

scrapy/utils/ftp.py

@@ -1,4 +1,5 @@
 import posixpath
+from contextlib import closing
 from ftplib import FTP, error_perm
 from posixpath import dirname
 from typing import IO
@@ -32,7 +33,7 @@ def ftp_store_file(
     """Opens a FTP connection with passed credentials,sets current directory
     to the directory extracted from given path, then uploads the file to server
     """
-    with FTP() as ftp:
+    with FTP() as ftp, closing(file):
         ftp.connect(host, port)
         ftp.login(username, password)
         if use_active_mode:
@@ -42,4 +43,3 @@ def ftp_store_file(
         ftp_makedirs_cwd(ftp, dirname)
         command = "STOR" if overwrite else "APPE"
         ftp.storbinary(f"{command} {filename}", file)
-        file.close()

scrapy/utils/log.py

@@ -149,11 +149,12 @@ def install_scrapy_root_handler(settings: Settings) -> None:
 def _uninstall_scrapy_root_handler() -> None:
     global _scrapy_root_handler  # noqa: PLW0603
 
-    if (
-        _scrapy_root_handler is not None
-        and _scrapy_root_handler in logging.root.handlers
-    ):
+    if _scrapy_root_handler is None:
+        return
+
+    if _scrapy_root_handler in logging.root.handlers:
         logging.root.removeHandler(_scrapy_root_handler)
+    _scrapy_root_handler.close()
     _scrapy_root_handler = None
 
 
@@ -247,8 +248,7 @@ def logformatter_adapter(
 ) -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]:
     """
     Helper that takes the dictionary output from the methods in LogFormatter
-    and adapts it into a tuple of positional arguments for logger.log calls,
-    handling backward compatibility as well.
+    and adapts it into a tuple of positional arguments for logger.log calls.
     """
 
     level = logkws.get("level", logging.INFO)

scrapy/utils/misc.py

@@ -135,14 +135,9 @@ def walk_modules(path: str) -> list[ModuleType]:  # pragma: no cover
     return list(walk_modules_iter(path))
 
 
-def md5sum(file: IO[bytes]) -> str:
+def md5sum(file: IO[bytes]) -> str:  # pragma: no cover
     """Calculate the md5 checksum of a file-like object without reading its
-    whole content in memory.
-
-    >>> from io import BytesIO
-    >>> md5sum(BytesIO(b'file content to hash'))
-    '784406af91dd5a54fbb9c84c2236595a'
-    """
+    whole content in memory."""
     warnings.warn(
         (
             "The scrapy.utils.misc.md5sum function is deprecated and will be "
@@ -162,7 +157,7 @@ def md5sum(file: IO[bytes]) -> str:
 
 def rel_has_nofollow(rel: str | None) -> bool:
     """Return True if link rel attribute has nofollow type"""
-    return rel is not None and "nofollow" in rel.replace(",", " ").split()
+    return rel is not None and "nofollow" in rel.lower().replace(",", " ").split()
 
 
 class SupportsFromCrawler(Protocol[_T_co, _P]):

scrapy/utils/reactor.py

@@ -187,7 +187,7 @@ def verify_installed_reactor(reactor_path: str) -> None:
 
 
 def verify_installed_asyncio_event_loop(loop_path: str) -> None:
-    """Raise :exc:`RuntimeError` if the even loop of the installed
+    """Raise :exc:`RuntimeError` if the event loop of the installed
     :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`
     does not match the specified import path or if no reactor is installed."""
     if not is_reactor_installed():

scrapy/utils/request.py

@@ -7,6 +7,7 @@ from __future__ import annotations
 
 import hashlib
 import json
+import logging
 from typing import TYPE_CHECKING, Any, Protocol
 from urllib.parse import urlunparse
 from weakref import WeakKeyDictionary
@@ -25,6 +26,9 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
     from scrapy.crawler import Crawler
+    from scrapy.http.request import CookiesT, VerboseCookie
+
+logger = logging.getLogger(__name__)
 
 
 _fingerprint_cache: WeakKeyDictionary[
@@ -179,11 +183,52 @@ def _get_method(obj: Any, name: Any) -> Any:
         raise ValueError(f"Method {name!r} not found in: {obj}") from None
 
 
+def _to_verbose_cookies(cookies: CookiesT) -> list[VerboseCookie]:
+    """Return a list of verbose cookies from ``request.cookies``.
+
+    The list of dicts form is returned as is, the dict one is converted first.
+    """
+    if isinstance(cookies, dict):
+        return [{"name": k, "value": v} for k, v in cookies.items()]
+    return cookies
+
+
+def _decode_cookie(cookie: VerboseCookie, request: Request) -> dict[str, str] | None:
+    """Return a dict with non-flag verbose cookie values converted to strings.
+
+    ``name``, ``value``, ``path``, ``domain`` are included, ``secure`` isn't.
+    """
+
+    decoded = {}
+    for key in ("name", "value", "path", "domain"):
+        value = cookie.get(key)
+        if value is None:
+            if key in {"name", "value"}:
+                logger.warning(
+                    f"Invalid cookie found in request {request}:"
+                    f" {cookie} ('{key}' is missing)"
+                )
+                return None
+            continue
+        if isinstance(value, (bool, float, int, str)):
+            decoded[key] = str(value)
+        else:
+            assert isinstance(value, bytes)
+            try:
+                decoded[key] = value.decode("utf8")
+            except UnicodeDecodeError:
+                logger.warning(
+                    f"Non UTF-8 encoded cookie found in request {request}: {cookie}",
+                )
+                decoded[key] = value.decode("latin1", errors="replace")
+    return decoded
+
+
 def request_to_curl(request: Request) -> str:
     """
     Converts a :class:`~scrapy.Request` object to a curl command.
 
-    :param :class:`~scrapy.Request`: Request object to be converted
+    :param request: Request object to be converted
     :return: string containing the curl command
     """
     method = request.method
@@ -195,17 +240,14 @@ def request_to_curl(request: Request) -> str:
     )
 
     url = request.url
-    cookies = ""
-    if request.cookies:
-        if isinstance(request.cookies, dict):
-            cookie = "; ".join(f"{k}={v}" for k, v in request.cookies.items())
-            cookies = f"--cookie '{cookie}'"
-        elif isinstance(request.cookies, list):
-            cookie = "; ".join(
-                f"{next(iter(c.keys()))}={next(iter(c.values()))}"
-                for c in request.cookies
-            )
-            cookies = f"--cookie '{cookie}'"
+
+    cookie_list: list[VerboseCookie] = _to_verbose_cookies(request.cookies)
+    pairs = [
+        f"{decoded['name']}={decoded['value']}"
+        for c in cookie_list
+        if (decoded := _decode_cookie(c, request)) is not None
+    ]
+    cookies = f"--cookie '{'; '.join(pairs)}'" if pairs else ""
 
     curl_cmd = f"curl -X {method} {url} {data} {headers} {cookies}".strip()
     return " ".join(curl_cmd.split())

scrapy/utils/response.py

@@ -88,7 +88,7 @@ def open_in_browser(
 
 
         def parse_details(self, response):
-            if "item name" not in response.body:
+            if "item name" not in response.text:
                 open_in_browser(response)
     """
     # circular imports

scrapy/utils/signal.py

@@ -39,7 +39,7 @@ def send_catch_log(
     *arguments: TypingAny,
     **named: TypingAny,
 ) -> list[tuple[TypingAny, TypingAny]]:
-    """Like ``pydispatcher.robust.sendRobust()`` but it also logs errors and returns
+    """Like ``pydispatch.robust.sendRobust()`` but it also logs errors and returns
     Failures instead of exceptions.
     """
     dont_log = named.pop("dont_log", ())
@@ -125,13 +125,11 @@ def _send_catch_log_deferred(
             **named,
         )
         d.addErrback(logerror, receiver)
-        # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html
+
         d2: Deferred[tuple[TypingAny, TypingAny]] = d.addBoth(
-            lambda result: (
-                receiver,  # pylint: disable=cell-var-from-loop  # noqa: B023
-                result,
-            )
+            lambda result, recv: (recv, result), receiver
         )
+
         dfds.append(d2)
 
     results = yield DeferredList(dfds)
@@ -174,9 +172,8 @@ async def _send_catch_log_asyncio(
 
     Returns a coroutine that completes once all signal handlers have finished.
 
-    This function requires
-    :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be
-    installed.
+    This function requires an installed asyncio reactor or a running asyncio
+    event loop.
 
     .. versionadded:: 2.14
     """

scrapy/utils/sitemap.py

@@ -97,10 +97,11 @@ class Sitemap:
 
     @staticmethod
     def _get_tag_name(elem: lxml.etree._Element) -> str:
-        if TYPE_CHECKING:
-            assert isinstance(elem.tag, str)
-        _, _, localname = elem.tag.partition("}")
-        return localname or elem.tag
+        tag = elem.tag
+        if not isinstance(tag, str):
+            return ""
+        _, _, localname = tag.partition("}")
+        return localname or tag
 
 
 def sitemap_urls_from_robots(

scrapy/utils/ssl.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import logging
 import ssl
+import warnings
 from typing import TYPE_CHECKING, Any, TypedDict, TypeVar
 
 import OpenSSL._util as pyOpenSSLutil
@@ -9,7 +10,11 @@ import OpenSSL.SSL
 import OpenSSL.version
 from twisted.internet.ssl import CertificateOptions, TLSVersion
 
-from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils._deps_compat import (
+    PYOPENSSL_X509_DEPRECATED,
+    TWISTED_TLS_LIMITS_OFFBY1,
+)
 from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
@@ -116,23 +121,42 @@ def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None:
 # pyOpenSSL utils
 
 
-def ffi_buf_to_string(buf: Any) -> str:
+def _ffi_buf_to_string(buf: Any) -> str:
     return to_unicode(pyOpenSSLutil.ffi.string(buf))
 
 
-def x509name_to_string(x509name: X509Name) -> str:
+def ffi_buf_to_string(buf: Any) -> str:  # pragma: no cover
+    warnings.warn(
+        "ffi_buf_to_string() is deprecated.",
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+    return ffi_buf_to_string(buf)
+
+
+def _x509name_to_string(x509name: X509Name) -> str:
     # from OpenSSL.crypto.X509Name.__repr__
+    # only used on pyOpenSSL < 24.3.0
     result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512)
     pyOpenSSLutil.lib.X509_NAME_oneline(
         x509name._name, result_buffer, len(result_buffer)
     )
-    return ffi_buf_to_string(result_buffer)
+    return _ffi_buf_to_string(result_buffer)
 
 
-def get_temp_key_info(ssl_object: Any) -> str | None:
+def x509name_to_string(x509name: X509Name) -> str:  # pragma: no cover
+    warnings.warn(
+        "x509name_to_string() is deprecated.",
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+    return _x509name_to_string(x509name)
+
+
+def _get_temp_key_info(ssl_object: Any) -> str | None:
     # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()
     if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"):
-        # removed in cryptography 40.0.0
+        # removed in cryptography 40.0.0 (required starting from pyOpenSSL 23.1.0)
         return None
     temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")
     if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p):
@@ -157,13 +181,22 @@ def get_temp_key_info(ssl_object: Any) -> str | None:
         cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid)
         if cname == pyOpenSSLutil.ffi.NULL:
             cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid)
-        key_info.append(ffi_buf_to_string(cname))
+        key_info.append(_ffi_buf_to_string(cname))
     else:
-        key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
+        key_info.append(_ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
     key_info.append(f"{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits")
     return ", ".join(key_info)
 
 
+def get_temp_key_info(ssl_object: Any) -> str | None:  # pragma: no cover
+    warnings.warn(
+        "get_temp_key_info() is deprecated. It's also a no-op with cryptography 40.0.0+.",
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+    return _get_temp_key_info(ssl_object)
+
+
 def get_openssl_version() -> str:
     system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)
     system_openssl = system_openssl_bytes.decode("ascii", errors="replace")
@@ -177,14 +210,21 @@ def _log_ssl_conn_debug_info(hostname: str, connection: OpenSSL.SSL.Connection)
         connection.get_protocol_version_name(),
         connection.get_cipher_name(),
     )
-    server_cert = connection.get_peer_certificate()
-    if server_cert:
-        logger.debug(
-            'SSL connection certificate: issuer "%s", subject "%s"',
-            x509name_to_string(server_cert.get_issuer()),
-            x509name_to_string(server_cert.get_subject()),
-        )
-    key_info = get_temp_key_info(connection._ssl)
+    if PYOPENSSL_X509_DEPRECATED:
+        if server_cert := connection.get_peer_certificate(as_cryptography=True):
+            logger.debug(
+                'SSL connection certificate: issuer "%s", subject "%s"',
+                server_cert.issuer.rfc4514_string(),
+                server_cert.subject.rfc4514_string(),
+            )
+    else:  # noqa: PLR5501
+        if server_cert_pyopenssl := connection.get_peer_certificate():
+            logger.debug(
+                'SSL connection certificate: issuer "%s", subject "%s"',
+                _x509name_to_string(server_cert_pyopenssl.get_issuer()),
+                _x509name_to_string(server_cert_pyopenssl.get_subject()),
+            )
+    key_info = _get_temp_key_info(connection._ssl)
     if key_info:
         logger.debug("SSL temp key: %s", key_info)
 

scrapy/utils/test.py

@@ -69,6 +69,7 @@ def get_crawler(
     # When needed, useful settings can be added here, e.g. ones that prevent
     # deprecation warnings.
     settings: dict[str, Any] = {
+        "TELNETCONSOLE_ENABLED": False,
         **get_reactor_settings(),
         **(settings_dict or {}),
     }

scrapy/utils/trackref.py

@@ -4,9 +4,7 @@ references to live object instances.
 If you want live objects for a particular class to be tracked, you only have to
 subclass from object_ref (instead of object).
 
-About performance: This library has a minimal performance impact when enabled,
-and no performance penalty at all when disabled (as object_ref becomes just an
-alias to object in that case).
+This library has a minimal performance impact.
 
 .. note:: PyPy uses a tracing garbage collector, so objects may
     remain in the ``live_refs`` longer than expected, even after they

scrapy/utils/url.py

@@ -117,8 +117,8 @@ def strip_url(
     - ``strip_credentials`` removes "user:password@"
     - ``strip_default_port`` removes ":80" (resp. ":443", ":21")
       from http:// (resp. https://, ftp://) URLs
-    - ``origin_only`` replaces path component with "/", also dropping
-      query and fragment components ; it also strips credentials
+    - ``origin_only`` replaces the  path component with "/", also dropping
+      the query component; it also strips credentials
     - ``strip_fragment`` drops any #fragment component
     """
 
@@ -139,7 +139,8 @@ def strip_url(
             ("ftp", 21),
         }
     ):
-        netloc = netloc.replace(f":{parsed_url.port}", "")
+        port_suffix = f":{parsed_url.port}"
+        netloc = netloc.removesuffix(port_suffix)
 
     return urlunparse(
         (

tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop_default.py

@@ -0,0 +1,31 @@
+import asyncio
+import sys
+
+from twisted.internet import asyncioreactor
+
+import scrapy
+from scrapy.crawler import AsyncCrawlerProcess
+
+if sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+loop = asyncio.SelectorEventLoop()
+asyncio.set_event_loop(loop)
+asyncioreactor.install(loop)
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = "no_request"
+
+    async def start(self):
+        return
+        yield
+
+
+process = AsyncCrawlerProcess(
+    settings={
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+        "ASYNCIO_EVENT_LOOP": "asyncio.SelectorEventLoop",
+    }
+)
+process.crawl(NoRequestsSpider)
+process.start()

tests/AsyncCrawlerProcess/dns_resolver_deprecated.py

@@ -0,0 +1,31 @@
+import sys
+
+import scrapy
+from scrapy.crawler import AsyncCrawlerProcess
+from scrapy.settings import Settings
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = "no_request"
+
+    async def start(self):
+        return
+        yield
+
+
+settings = Settings()
+# The deprecated DNS_RESOLVER setting, set above its default priority so that
+# AsyncCrawlerProcess._setup_reactor() emits the deprecation warning.
+settings.set("DNS_RESOLVER", "scrapy.resolver.CachingThreadedResolver", priority=10)
+if len(sys.argv) > 1 and sys.argv[1] == "twisted-wins":
+    # TWISTED_DNS_RESOLVER at a higher priority takes precedence over the
+    # deprecated DNS_RESOLVER setting.
+    settings.set(
+        "TWISTED_DNS_RESOLVER",
+        "scrapy.resolver.CachingThreadedResolver",
+        priority=20,
+    )
+
+process = AsyncCrawlerProcess(settings)
+process.crawl(NoRequestsSpider)
+process.start()

tests/AsyncCrawlerProcess/reactorless_sleeping.py

@@ -17,4 +17,4 @@ class SleepingSpider(scrapy.Spider):
 process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
 
 process.crawl(SleepingSpider)
-process.start()
+process.start(stop_after_crawl="--no-stop" not in sys.argv)

tests/AsyncCrawlerProcess/sleeping.py

@@ -17,4 +17,4 @@ class SleepingSpider(scrapy.Spider):
 process = AsyncCrawlerProcess(settings={})
 
 process.crawl(SleepingSpider)
-process.start()
+process.start(stop_after_crawl="--no-stop" not in sys.argv)

tests/CrawlerProcess/dns_resolver_deprecated.py

@@ -0,0 +1,31 @@
+import sys
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from scrapy.settings import Settings
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = "no_request"
+
+    async def start(self):
+        return
+        yield
+
+
+settings = Settings()
+# The deprecated DNS_RESOLVER setting, set above its default priority so that
+# CrawlerProcess._setup_reactor() emits the deprecation warning.
+settings.set("DNS_RESOLVER", "scrapy.resolver.CachingThreadedResolver", priority=10)
+if len(sys.argv) > 1 and sys.argv[1] == "twisted-wins":
+    # TWISTED_DNS_RESOLVER at a higher priority takes precedence over the
+    # deprecated DNS_RESOLVER setting.
+    settings.set(
+        "TWISTED_DNS_RESOLVER",
+        "scrapy.resolver.CachingThreadedResolver",
+        priority=20,
+    )
+
+process = CrawlerProcess(settings)
+process.crawl(NoRequestsSpider)
+process.start()