Merge remote-tracking branch 'origin/master' into httpcache-inflight

.github/workflows/flag-prs-for-triage.yml

@@ -20,6 +20,12 @@ name: Flag PRs for triage
 #   lists, em dash density or stock acknowledgement phrases.
 # - Agent branch: the branch name carries an agent prefix.
 #
+# Authors that the organisations behind this repository already trust are left
+# alone before any of that runs: public members of those organisations, and
+# authors with a track record of pull requests merged into their repositories.
+# Trust from a merge record rather than from a list of names keeps the exemption
+# in step with who is actually contributing.
+#
 # Deliberately not used: account age, fork age, follower count, total pull
 # request count and cross-repository merge ratio. All of them were measured
 # against hand-labelled pull requests and either failed to separate or, in the
@@ -56,6 +62,8 @@ jobs:
             const MAX_REPOS_PER_WEEK = 2;
             const VOICE = { structure: 0.10, emDashPerKChar: 0.30, acknowledgement: 0.40 };
             const EVENT_PAGES = 3;
+            const TRUSTED_ORGS = ['scrapy', 'scrapy-plugins', 'scrapinghub', 'zytedata'];
+            const MIN_TRUSTED_MERGES = 10;
             const AGENT_BRANCH = /^(agent|codex|claude|cursor|devin|copilot|jules|bot)[\/_-]/i;
 
             const { owner, repo } = context.repo;
@@ -93,6 +101,33 @@ jobs:
               throw error;
             });
 
+            // author_association only reports membership of the organisation
+            // that owns this repository, and only when it is public, so trust
+            // in the author is established here instead.
+            const trustedOrg = (await Promise.all(TRUSTED_ORGS.map(org =>
+              orNull(withRetries(`Checking public membership of ${org}`, () =>
+                github.rest.orgs.checkPublicMembershipForUser({ org, username: author }),
+              )).then(response => response && org),
+            ))).find(Boolean);
+            if (trustedOrg) {
+              core.info(`Skipping PR #${pr.number} by ${author} (public member of ${trustedOrg}).`);
+              return;
+            }
+            // Repeating a qualifier narrows the search instead of widening it,
+            // hence the explicit disjunction.
+            const trustedMerges = await orNull(withRetries('Counting merged PRs in trusted organisations', () =>
+              github.rest.search.issuesAndPullRequests({
+                q: `author:${author} type:pr is:merged`
+                  + ` (${TRUSTED_ORGS.map(org => `org:${org}`).join(' OR ')})`,
+                advanced_search: 'true', per_page: 1,
+              }).then(response => response.data.total_count),
+            ));
+            if (trustedMerges >= MIN_TRUSTED_MERGES) {
+              core.info(`Skipping PR #${pr.number} by ${author}`
+                + ` (${trustedMerges} PR(s) merged into ${TRUSTED_ORGS.join(', ')}).`);
+              return;
+            }
+
             const opened = new Date(pr.created_at);
             const daysBefore = date => (opened - new Date(date)) / 86400000;
 

.github/workflows/tests-ubuntu.yml

@@ -86,6 +86,7 @@ jobs:
         - python-version: pypy3.11-7.3.20
           env:
             TOXENV: pypy3-extra-deps
+          coverage: true
         - python-version: "3.14"
           env:
             TOXENV: botocore

.github/workflows/tests-vcs-deps.yml

@@ -0,0 +1,56 @@
+name: VCS dependencies
+
+permissions:
+  contents: read
+
+on:
+  schedule:
+  - cron: '0 4 * * *'
+  workflow_dispatch:
+
+concurrency:
+  group: ${{github.workflow}}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tests:
+    name: tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    env:
+      # A development branch of a dependency can make a test hang forever, so
+      # tests get a time limit here that they do not need elsewhere.
+      PYTEST_ADDOPTS: -n auto --no-cov --timeout=120
+      TOXENV: vcs-deps
+      UV_PYTHON_PREFERENCE: only-system
+    steps:
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      with:
+        persist-credentials: false
+
+    - name: Set up Python
+      uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+      with:
+        python-version: "3.14"
+
+    # Dependencies that ship wheels on PyPI are built from source here, so
+    # their build dependencies are needed: libxml2 and libxslt for lxml,
+    # libjpeg and zlib for Pillow, and autotools for the libuv bundled in
+    # uvloop.
+    - name: Install system libraries
+      run: |
+        sudo apt-get update
+        sudo apt-get install automake libjpeg-dev libtool libxml2-dev libxslt-dev zlib1g-dev
+
+    - name: Set up uv
+      uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+      with:
+        cache-dependency-glob: |
+          pyproject.toml
+          tox.ini
+
+    - name: Install mitmproxy
+      run: uv tool install --python cpython mitmproxy
+
+    - name: Run tests
+      run: uvx --with tox-uv tox

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
   hooks:
     - id: sphinx-lint
 - repo: https://github.com/scrapy/sphinx-scrapy
-  rev: 0.8.10
+  rev: 0.8.11
   hooks:
     - id: sphinx-scrapy
 - repo: https://github.com/zizmorcore/zizmor-pre-commit

docs/_ext/scrapydocs.py

@@ -137,14 +137,6 @@ def source_role(
     return [node], []
 
 
-def issue_role(
-    name, rawtext, text: str, lineno, inliner, options=None, content=None
-) -> tuple[list[Any], list[Any]]:
-    ref = "https://github.com/scrapy/scrapy/issues/" + text
-    node = nodes.reference(rawtext, "issue " + text, refuri=ref)
-    return [node], []
-
-
 def commit_role(
     name, rawtext, text: str, lineno, inliner, options=None, content=None
 ) -> tuple[list[Any], list[Any]]:
@@ -164,7 +156,6 @@ def rev_role(
 def setup(app: Sphinx) -> dict[str, Any]:
     app.add_role("source", source_role)
     app.add_role("commit", commit_role)
-    app.add_role("issue", issue_role)
     app.add_role("rev", rev_role)
 
     app.add_node(

docs/conf.py

@@ -31,9 +31,14 @@ extensions = [
     "sphinx_scrapy",
     "scrapyfixautodoc",  # Must be after "sphinx.ext.autodoc"
     "sphinx.ext.coverage",
+    "sphinx_reredirects",
     "sphinx_rtd_dark_mode",
 ]
 
+redirects = {
+    "topics/broad-crawls": "optimize.html#broad-crawls",
+}
+
 templates_path = ["_templates"]
 exclude_patterns = ["build", "Thumbs.db", ".DS_Store"]
 

docs/faq.rst

@@ -141,7 +141,7 @@ middleware with a :ref:`custom downloader middleware
 
 -   If you can meet the installation requirements, use pyre2_ instead of
     Python’s re_ to compile your URL-filtering regular expression. See
-    :issue:`1908`.
+    :gh:`1908`.
 
 See also `other suggestions at StackOverflow
 <https://stackoverflow.com/q/36440681>`__.
@@ -292,7 +292,7 @@ Does Scrapy manage cookies automatically?
 Yes, Scrapy receives and keeps track of cookies sent by servers, and sends them
 back on subsequent requests, like any regular web browser does.
 
-For more info see :ref:`topics-request-response` and :ref:`cookies-mw`.
+For more info see :ref:`cookies`.
 
 How can I see the cookies being sent and received from Scrapy?
 --------------------------------------------------------------
@@ -419,7 +419,7 @@ Running ``runspider`` I get ``error: No spider found in file: <filename>``
 This may happen if your Scrapy project has a spider module with a name that
 conflicts with the name of one of the `Python standard library modules`_, such
 as ``csv.py`` or ``os.py``, or any `Python package`_ that you have installed.
-See :issue:`2680`.
+See :gh:`2680`.
 
 
 .. _has been reported: https://github.com/scrapy/scrapy/issues/2905

docs/index.rst

@@ -78,6 +78,7 @@ Basic concepts
    topics/item-pipeline
    topics/feed-exports
    topics/request-response
+   topics/cookies
    topics/link-extractors
    topics/settings
    topics/exceptions
@@ -109,6 +110,9 @@ Basic concepts
 :doc:`topics/request-response`
     Understand the classes used to represent HTTP requests and responses.
 
+:doc:`topics/cookies`
+    Send and receive cookies.
+
 :doc:`topics/link-extractors`
     Convenient classes to extract links to follow from pages.
 
@@ -152,7 +156,7 @@ Solving specific problems
    topics/contracts
    topics/practices
    topics/security
-   topics/broad-crawls
+   topics/optimize
    topics/developer-tools
    topics/dynamic-content
    topics/leaks
@@ -180,8 +184,8 @@ Solving specific problems
     Understand the security implications of Scrapy defaults and how to harden
     them.
 
-:doc:`topics/broad-crawls`
+:doc:`topics/optimize`
-    Tune Scrapy for crawling a lot domains in parallel.
+    Find the bottleneck of your crawls and learn how to address it.
 
 :doc:`topics/developer-tools`
     Learn how to scrape with your browser's developer tools.

docs/intro/install.rst

@@ -111,8 +111,6 @@ The following extras are available:
      - Provides
    * - ``bpython``
      - :ref:`bpython shell <shell-config>`
-   * - ``brotli``
-     - :ref:`Brotli response decompression <http-compression>`
    * - ``gcs``
      - :ref:`Google Cloud Storage <topics-feed-storage-gcs>` for
        :ref:`feed exports <topics-feed-exports>` and

docs/intro/tutorial.rst

@@ -72,6 +72,11 @@ This will create a ``tutorial`` directory with the following contents::
             spiders/          # a directory where you'll later put your spiders
                 __init__.py
 
+Before crawling anything, open ``settings.py`` and uncomment the
+:setting:`USER_AGENT` line to identify yourself, e.g. a project name plus a URL
+or an email address. Website owners who take issue with your crawler can then
+ask you to adjust it, rather than block it.
+
 
 Our first Spider
 ================

docs/requirements.in

@@ -3,6 +3,7 @@ pydantic
 scrapy-spider-metadata
 sphinx
 sphinx-notfound-page
+sphinx-reredirects
 sphinx-rtd-theme
 sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.11

docs/requirements.txt

@@ -134,6 +134,7 @@ sphinx==9.1.0
     #   sphinx-llms-txt
     #   sphinx-markdown-builder
     #   sphinx-notfound-page
+    #   sphinx-reredirects
     #   sphinx-rtd-theme
     #   sphinx-scrapy
     #   sphinxcontrib-jquery
@@ -147,13 +148,15 @@ sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builde
     # via sphinx-scrapy
 sphinx-notfound-page==1.1.0
     # via -r docs/requirements.in
+sphinx-reredirects==1.1.0
+    # via -r docs/requirements.in
 sphinx-rtd-dark-mode==1.3.0
     # via -r docs/requirements.in
 sphinx-rtd-theme==3.1.0
     # via
     #   -r docs/requirements.in
     #   sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@6f8e5e0bbd171a857da480f7188f2a205041cb60
     # via -r docs/requirements.in
 sphinx-sitemap==2.9.0
     # via sphinx-scrapy

docs/topics/api.rst

@@ -35,6 +35,13 @@ how you :ref:`configure the downloader middlewares
     :class:`scrapy.Spider` subclass and a
     :class:`scrapy.settings.Settings` object.
 
+    The :attr:`engine`, :attr:`extensions`, :attr:`logformatter`,
+    :attr:`request_fingerprinter` and :attr:`stats` attributes get their value
+    when the crawl starts, and raise :exc:`RuntimeError` when read before that.
+
+    .. versionchanged:: VERSION
+        Those attributes used to be ``None`` before getting their value.
+
     .. attribute:: request_fingerprinter
 
         The request fingerprint builder of this crawler.

docs/topics/broad-crawls.rst

@@ -1,194 +0,0 @@
-.. _topics-broad-crawls:
-
-============
-Broad Crawls
-============
-
-Scrapy defaults are optimized for crawling specific sites. These sites are
-often handled by a single Scrapy spider, although this is not necessary or
-required (for example, there are generic spiders that handle any given site
-thrown at them).
-
-In addition to this "focused crawl", there is another common type of crawling
-which covers a large (potentially unlimited) number of domains, and is only
-limited by time or other arbitrary constraint, rather than stopping when the
-domain was crawled to completion or when there are no more requests to perform.
-These are called "broad crawls" and is the typical crawlers employed by search
-engines.
-
-These are some common properties often found in broad crawls:
-
-* they crawl many domains (often, unbounded) instead of a specific set of sites
-
-* they don't necessarily crawl domains to completion, because it would be
-  impractical (or impossible) to do so, and instead limit the crawl by time or
-  number of pages crawled
-
-* they are simpler in logic (as opposed to very complex spiders with many
-  extraction rules) because data is often post-processed in a separate stage
-
-* they crawl many domains concurrently, which allows them to achieve faster
-  crawl speeds by not being limited by any particular site constraint (each site
-  is crawled slowly to respect politeness, but many sites are crawled in
-  parallel)
-
-As said above, Scrapy default settings are optimized for focused crawls, not
-broad crawls. However, due to its asynchronous architecture, Scrapy is very
-well suited for performing fast broad crawls. This page summarizes some things
-you need to keep in mind when using Scrapy for doing broad crawls, along with
-concrete suggestions of Scrapy settings to tune in order to achieve an
-efficient broad crawl.
-
-.. _broad-crawls-scheduler-priority-queue:
-
-.. _broad-crawls-concurrency:
-
-Increase concurrency
-====================
-
-Concurrency is the number of requests that are processed in parallel. There is
-a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that
-can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`).
-
-The default global concurrency limit in Scrapy is not suitable for crawling
-many different domains in parallel, so you will want to increase it. How much
-to increase it will depend on how much CPU and memory your crawler will have
-available.
-
-A good starting point is ``100``:
-
-.. code-block:: python
-
-    CONCURRENT_REQUESTS = 100
-
-But the best way to find out is by doing some trials and identifying at what
-concurrency your Scrapy process gets CPU bounded. For optimum performance, you
-should pick a concurrency where CPU usage is at 80-90%.
-
-Increasing concurrency also increases memory usage. If memory usage is a
-concern, you might need to lower your global concurrency limit accordingly.
-
-
-Increase Twisted IO thread pool maximum size
-============================================
-
-Currently Scrapy does DNS resolution in a blocking way with usage of thread
-pool. With higher concurrency levels the crawling could be slow or even fail
-hitting DNS resolver timeouts. Possible solution to increase the number of
-threads handling DNS queries. The DNS queue will be processed faster speeding
-up establishing of connection and crawling overall.
-
-To increase maximum thread pool size use:
-
-.. code-block:: python
-
-    REACTOR_THREADPOOL_MAXSIZE = 20
-
-Setup your own DNS
-==================
-
-If you have multiple crawling processes and single central DNS, it can act
-like DoS attack on the DNS server resulting to slow down of entire network or
-even blocking your machines. To avoid this setup your own DNS server with
-local cache and upstream to some large DNS like OpenDNS or Verizon.
-
-Reduce log level
-================
-
-When doing broad crawls you are often only interested in the crawl rates you
-get and any errors found. These stats are reported by Scrapy when using the
-``INFO`` log level. In order to save CPU (and log storage requirements) you
-should not use ``DEBUG`` log level when performing large broad crawls in
-production. Using ``DEBUG`` level when developing your (broad) crawler may be
-fine though.
-
-To set the log level use:
-
-.. code-block:: python
-
-    LOG_LEVEL = "INFO"
-
-Disable cookies
-===============
-
-Disable cookies unless you *really* need. Cookies are often not needed when
-doing broad crawls (search engine crawlers ignore them), and they improve
-performance by saving some CPU cycles and reducing the memory footprint of your
-Scrapy crawler.
-
-To disable cookies use:
-
-.. code-block:: python
-
-    COOKIES_ENABLED = False
-
-Disable retries
-===============
-
-Retrying failed HTTP requests can slow down the crawls substantially, especially
-when sites causes are very slow (or fail) to respond, thus causing a timeout
-error which gets retried many times, unnecessarily, preventing crawler capacity
-to be reused for other domains.
-
-To disable retries use:
-
-.. code-block:: python
-
-    RETRY_ENABLED = False
-
-Reduce download timeout
-=======================
-
-Unless you are crawling from a very slow connection (which shouldn't be the
-case for broad crawls) reduce the download timeout so that stuck requests are
-discarded quickly and free up capacity to process the next ones.
-
-To reduce the download timeout use:
-
-.. code-block:: python
-
-    DOWNLOAD_TIMEOUT = 15
-
-Disable redirects
-=================
-
-Consider disabling redirects, unless you are interested in following them. When
-doing broad crawls it's common to save redirects and resolve them when
-revisiting the site at a later crawl. This also help to keep the number of
-request constant per crawl batch, otherwise redirect loops may cause the
-crawler to dedicate too many resources on any specific domain.
-
-To disable redirects use:
-
-.. code-block:: python
-
-    REDIRECT_ENABLED = False
-
-.. _broad-crawls-bfo:
-
-Crawl in BFO order
-==================
-
-:ref:`Scrapy crawls in DFO order by default <faq-bfo-dfo>`.
-
-In broad crawls, however, page crawling tends to be faster than page
-processing. As a result, unprocessed early requests stay in memory until the
-final depth is reached, which can significantly increase memory usage.
-
-:ref:`Crawl in BFO order <faq-bfo-dfo>` instead to save memory.
-
-
-Be mindful of memory leaks
-==========================
-
-If your broad crawl shows a high memory usage, in addition to :ref:`crawling in
-BFO order <broad-crawls-bfo>` and :ref:`lowering concurrency
-<broad-crawls-concurrency>` you should :ref:`debug your memory leaks
-<topics-leaks>`.
-
-
-Install a specific Twisted reactor
-==================================
-
-If the crawl is exceeding the system's capabilities, you might want to try
-installing a specific Twisted reactor, via the :setting:`TWISTED_REACTOR` setting.

docs/topics/cookies.rst

@@ -0,0 +1,138 @@
+.. _cookies:
+.. _cookies-mw:
+
+=======
+Cookies
+=======
+
+Scrapy keeps track of the cookies that websites set and sends them back on
+later requests to those websites, just like a web browser does. That is the job
+of :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`, which is
+enabled by default.
+
+
+Setting cookies on a request
+============================
+
+.. invisible-code-block: python
+
+    from scrapy import Request
+
+Use the ``cookies`` parameter of :class:`~scrapy.Request` to send cookies of
+your own, either as a dict:
+
+.. code-block:: python
+
+    request = Request(
+        url="https://example.com",
+        cookies={"currency": "USD", "country": "UY"},
+    )
+
+Or as a list of dicts, which also lets you set cookie attributes:
+
+.. code-block:: python
+
+    request = Request(
+        url="https://example.com",
+        cookies=[
+            {
+                "name": "currency",
+                "value": "USD",
+                "domain": "example.com",
+                "path": "/currency",
+                "secure": True,
+            },
+        ],
+    )
+
+Setting attributes is only useful if the cookies are stored for later requests,
+i.e. if :reqmeta:`dont_merge_cookies` is not enabled.
+
+.. caution:: Cookies set through the ``Cookie`` header are not handled by
+    :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`, which
+    drops that header.
+
+.. caution:: When a cookie name or value is a byte sequence that is not UTF-8
+    encoded, the cookie is dropped and a warning is logged. See
+    :ref:`topics-logging-advanced-customization` to customize the logging
+    behavior.
+
+
+.. reqmeta:: cookiejar
+
+Multiple cookie sessions per spider
+===================================
+
+By default all requests share a single cookie jar (session). To use different
+ones, pass an identifier in the :reqmeta:`cookiejar` request meta key:
+
+.. skip: next
+.. code-block:: python
+
+    for i, url in enumerate(urls):
+        yield Request(url, meta={"cookiejar": i}, callback=self.parse_page)
+
+The :reqmeta:`cookiejar` meta key is not "sticky", so you need to keep passing
+it along on subsequent requests:
+
+.. code-block:: python
+
+    def parse_page(self, response):
+        return Request(
+            "https://example.com/otherpage",
+            meta={"cookiejar": response.meta["cookiejar"]},
+            callback=self.parse_other_page,
+        )
+
+
+.. reqmeta:: dont_merge_cookies
+
+Skipping the cookie jar for a request
+=====================================
+
+Set the :reqmeta:`dont_merge_cookies` request meta key to ``True`` to keep a
+request from touching the cookie jar in either direction: no stored cookie is
+sent with the request, and no cookie received in the response is stored. The
+cookies of the request itself are ignored as well.
+
+
+.. setting:: COOKIES_ENABLED
+
+COOKIES_ENABLED
+===============
+
+Default: ``True``
+
+Whether to enable :class:`~scrapy.downloadermiddlewares.cookies.CookiesMiddleware`.
+If disabled, no cookies are sent to web servers.
+
+
+.. setting:: COOKIES_DEBUG
+
+COOKIES_DEBUG
+=============
+
+Default: ``False``
+
+If enabled, Scrapy logs all cookies sent in requests (i.e. the ``Cookie``
+header) and all cookies received in responses (i.e. the ``Set-Cookie``
+header)::
+
+    2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened
+    2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: <GET http://www.diningcity.com/netherlands/index.html>
+            Cookie: clientlanguage_nl=en_EN
+    2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html>
+            Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/
+            Set-Cookie: ip_isocode=US
+            Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/
+    2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://www.diningcity.com/netherlands/index.html> (referer: None)
+    [...]
+
+
+CookiesMiddleware
+=================
+
+.. module:: scrapy.downloadermiddlewares.cookies
+   :synopsis: Cookies Downloader Middleware
+
+.. autoclass:: CookiesMiddleware

docs/topics/download-handlers.rst

@@ -130,17 +130,23 @@ using different handlers.
 Here is a comparison of some features of the built-in HTTP handlers, see the
 individual handler docs for more differences:
 
-================== ================= ===================== ====================
+=================== ================= ===================== ====================
-Feature            H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler
+Feature             H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler
-================== ================= ===================== ====================
+=================== ================= ===================== ====================
-Requires asyncio   No                No                    Yes
+Requires asyncio    No                No                    Yes
-Requires a reactor Yes               Yes                   No
+Requires a reactor  Yes               Yes                   No
-HTTP/1.1           No                Yes                   Yes
+HTTP/1.1            No                Yes                   Yes
-HTTP/2             Yes               No                    Yes
+HTTP/2              Yes               No                    Yes
-TLS implementation ``cryptography``  ``cryptography``      Stdlib ``ssl``
+TLS implementation  ``cryptography``  ``cryptography``      Stdlib ``ssl``
-HTTP proxies       No                Yes                   Yes
+HTTP proxies        No                Yes                   Yes
-SOCKS proxies      No                No                    Yes
+SOCKS proxies       No                No                    Yes
-================== ================= ===================== ====================
+Bad header handling Not applicable    Skip bad              Fail
+=================== ================= ===================== ====================
+
+Bad header handling is what a handler does when a response has a bad header
+line, e.g. one with no colon in it, which some servers send. Handlers that skip
+bad header lines, like web browsers do, still parse the header lines that follow
+them; other handlers also lose those, or cannot download such responses at all.
 
 You can find additional HTTP download handlers in the
 scrapy-download-handlers-incubator_ package. This package is made by the Scrapy
@@ -191,6 +197,7 @@ Features and limitations
 HTTP proxies                No (not implemented)
 SOCKS proxies               No (not supported by the library)
 HTTP/2                      Yes
+Bad header handling         Not applicable (HTTP/2 only)
 ``response.certificate``    :class:`twisted.internet.ssl.Certificate` object
 Per-request ``bindaddress`` Yes
 TLS implementation          ``pyOpenSSL``/``cryptography``
@@ -203,9 +210,6 @@ Other limitations:
 -   IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`
     to ``scrapy.resolver.CachingHostnameResolver``.
 
--   No support for the :signal:`bytes_received` and :signal:`headers_received`
-    signals.
-
 Known limitations of the HTTP/2 support:
 
 -   No support for HTTP/2 Cleartext (h2c), since no major browser supports
@@ -242,11 +246,16 @@ Features and limitations
 HTTP proxies                Yes
 SOCKS proxies               No (not supported by the library)
 HTTP/2                      No (implemented as a separate handler)
+Bad header handling         Skip bad, like web browsers do
 ``response.certificate``    :class:`twisted.internet.ssl.Certificate` object
 Per-request ``bindaddress`` Yes
 TLS implementation          ``pyOpenSSL``/``cryptography``
 =========================== ================================================
 
+.. versionchanged:: VERSION
+   Bad header lines with no colon in them are now skipped, instead of making
+   the whole response impossible to download.
+
 Other limitations:
 
 -   IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`
@@ -300,6 +309,7 @@ Features and limitations
 HTTP proxies                Yes
 SOCKS proxies               Yes (SOCKS5)
 HTTP/2                      Yes
+Bad header handling         Fail (not supported by the library)
 ``response.certificate``    DER bytes
 Per-request ``bindaddress`` No (not supported by the library)
 TLS implementation          Standard library ``ssl``

docs/topics/downloader-middleware.rst

@@ -156,6 +156,61 @@ defines one or more of these methods:
       :param exception: the raised exception
       :type exception: an ``Exception`` object
 
+.. _mw-download:
+
+Downloading a request from a downloader middleware
+==================================================
+
+A downloader middleware can download a request of its own while it processes
+another one, e.g. to fetch something that the request it is processing needs.
+The built-in :ref:`robots.txt middleware <topics-dlmw-robots>` does that: it
+holds each request while it downloads the ``robots.txt`` file of its website.
+
+Use :meth:`crawler.engine.download_async()
+<scrapy.core.engine.ExecutionEngine.download_async>` for that:
+
+.. code-block:: python
+
+    from scrapy import Request
+    from scrapy.http.request import NO_CALLBACK
+
+
+    class TokenMiddleware:
+        def __init__(self, crawler):
+            self.crawler = crawler
+            self.token = None
+
+        @classmethod
+        def from_crawler(cls, crawler):
+            return cls(crawler)
+
+        async def process_request(self, request):
+            if request.meta.get("dont_obey_robotstxt"):
+                return
+            if self.token is None:
+                response = await self.crawler.engine.download_async(
+                    Request(
+                        "https://example.com/token",
+                        callback=NO_CALLBACK,
+                        meta={"dont_obey_robotstxt": True},
+                    )
+                )
+                self.token = response.text
+            request.headers["Authorization"] = self.token
+
+Requests that you download this way go through the downloader middleware chain
+as well, including your own middleware and the :ref:`robots.txt middleware
+<topics-dlmw-robots>`, which holds a request until the ``robots.txt`` file of
+its website arrives. Be careful not to introduce deadlocks: a request that you
+download must not end up waiting for the request that is waiting for it. Hence
+:reqmeta:`dont_obey_robotstxt` above, which makes both middlewares let the token
+request through.
+
+While the first token response is in transit, ``process_request`` runs for other
+requests as well, and the middleware above downloads a token for each of them.
+Cache the task that downloads the token, and not only its result, to download
+the token only once.
+
 .. _topics-downloader-middleware-ref:
 
 Built-in downloader middleware reference
@@ -169,106 +224,10 @@ middleware, see the :ref:`downloader middleware usage guide
 For a list of the components enabled by default (and their orders) see the
 :setting:`DOWNLOADER_MIDDLEWARES_BASE` setting.
 
-.. _cookies-mw:
-
 CookiesMiddleware
 -----------------
 
-.. module:: scrapy.downloadermiddlewares.cookies
+See :ref:`cookies`.
-   :synopsis: Cookies Downloader Middleware
-
-.. class:: CookiesMiddleware
-
-   This middleware enables working with sites that require cookies, such as
-   those that use sessions. It keeps track of cookies sent by web servers, and
-   sends them back on subsequent requests (from that spider), just like web
-   browsers do.
-
-   .. caution:: When non-UTF8 encoded byte sequences are passed to a
-      :class:`~scrapy.Request`, the ``CookiesMiddleware`` will log
-      a warning. Refer to :ref:`topics-logging-advanced-customization`
-      to customize the logging behaviour.
-
-   .. caution:: Cookies set via the ``Cookie`` header are not considered by the
-      :ref:`cookies-mw`. If you need to set cookies for a request, use the
-      :class:`Request.cookies <scrapy.Request>` parameter. This is a known
-      current limitation that is being worked on.
-
-The following settings can be used to configure the cookie middleware:
-
-* :setting:`COOKIES_ENABLED`
-* :setting:`COOKIES_DEBUG`
-
-.. reqmeta:: cookiejar
-
-Multiple cookie sessions per spider
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-There is support for keeping multiple cookie sessions per spider by using the
-:reqmeta:`cookiejar` Request meta key. By default it uses a single cookie jar
-(session), but you can pass an identifier to use different ones.
-
-For example:
-
-.. skip: next
-.. code-block:: python
-
-    for i, url in enumerate(urls):
-        yield scrapy.Request(url, meta={"cookiejar": i}, callback=self.parse_page)
-
-Keep in mind that the :reqmeta:`cookiejar` meta key is not "sticky". You need to keep
-passing it along on subsequent requests. For example:
-
-.. code-block:: python
-
-    def parse_page(self, response):
-        # do some processing
-        return scrapy.Request(
-            "http://www.example.com/otherpage",
-            meta={"cookiejar": response.meta["cookiejar"]},
-            callback=self.parse_other_page,
-        )
-
-.. setting:: COOKIES_ENABLED
-
-COOKIES_ENABLED
-~~~~~~~~~~~~~~~
-
-Default: ``True``
-
-Whether to enable the cookies middleware. If disabled, no cookies will be sent
-to web servers.
-
-Notice that despite the value of :setting:`COOKIES_ENABLED` setting if
-``Request.``:reqmeta:`meta['dont_merge_cookies'] <dont_merge_cookies>`
-evaluates to ``True`` the request cookies will **not** be sent to the
-web server and received cookies in :class:`~scrapy.http.Response` will
-**not** be merged with the existing cookies.
-
-For more detailed information see the ``cookies`` parameter in
-:class:`~scrapy.Request`.
-
-.. setting:: COOKIES_DEBUG
-
-COOKIES_DEBUG
-~~~~~~~~~~~~~
-
-Default: ``False``
-
-If enabled, Scrapy will log all cookies sent in requests (i.e. ``Cookie``
-header) and all cookies received in responses (i.e. ``Set-Cookie`` header).
-
-Here's an example of a log with :setting:`COOKIES_DEBUG` enabled::
-
-    2011-04-06 14:35:10-0300 [scrapy.core.engine] INFO: Spider opened
-    2011-04-06 14:35:10-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Sending cookies to: <GET http://www.diningcity.com/netherlands/index.html>
-            Cookie: clientlanguage_nl=en_EN
-    2011-04-06 14:35:14-0300 [scrapy.downloadermiddlewares.cookies] DEBUG: Received cookies from: <200 http://www.diningcity.com/netherlands/index.html>
-            Set-Cookie: JSESSIONID=B~FA4DC0C496C8762AE4F1A620EAB34F38; Path=/
-            Set-Cookie: ip_isocode=US
-            Set-Cookie: clientlanguage_nl=en_EN; Expires=Thu, 07-Apr-2011 21:21:34 GMT; Path=/
-    2011-04-06 14:49:50-0300 [scrapy.core.engine] DEBUG: Crawled (200) <GET http://www.diningcity.com/netherlands/index.html> (referer: None)
-    [...]
 
 
 DefaultHeadersMiddleware
@@ -747,14 +706,13 @@ HttpCompressionMiddleware
 
 .. class:: HttpCompressionMiddleware
 
-   This middleware allows compressed (gzip, deflate) traffic to be
+   This middleware allows compressed (gzip, deflate, `brotli`_) traffic to be
    sent/received from web sites.
 
-   This middleware also supports decoding `brotli-compressed`_ responses with
+   This middleware also supports decoding `zstd-compressed`_ responses with
-   the :ref:`brotli <extras>` extra, and `zstd-compressed`_
+   the :ref:`zstd <extras>` extra.
-   responses with the :ref:`zstd <extras>` extra.
 
-.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
+.. _brotli: https://www.ietf.org/rfc/rfc7932.txt
 .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
 
 
@@ -847,40 +805,9 @@ OffsiteMiddleware
 .. module:: scrapy.downloadermiddlewares.offsite
    :synopsis: Offsite Middleware
 
-.. class:: OffsiteMiddleware
+.. autoclass:: OffsiteMiddleware
 
-   .. versionadded:: 2.11.2
+   .. automethod:: should_follow
-
-   Filters out Requests for URLs outside the domains covered by the spider.
-
-   This middleware filters out every request whose host names aren't in the
-   spider's :attr:`~scrapy.Spider.allowed_domains` attribute.
-   All subdomains of any domain in the list are also allowed.
-   E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``
-   but not ``www2.example.com`` nor ``example.com``.
-
-   When your spider returns a request for a domain not belonging to those
-   covered by the spider, this middleware will log a debug message similar to
-   this one::
-
-      DEBUG: Filtered offsite request to 'offsite.example': <GET http://offsite.example/some/page.html>
-
-   To avoid filling the log with too much noise, it will only print one of
-   these messages for each new domain filtered. So, for example, if another
-   request for ``offsite.example`` is filtered, no log message will be
-   printed. But if a request for ``other.example`` is filtered, a message
-   will be printed (but only for the first request filtered).
-
-   If the spider doesn't define an
-   :attr:`~scrapy.Spider.allowed_domains` attribute, or the
-   attribute is empty, the offsite middleware will allow all requests.
-
-   .. reqmeta:: allow_offsite
-
-   If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to
-   ``True`` or :attr:`Request.meta <scrapy.Request.meta>` has ``allow_offsite``
-   set to ``True``, then the OffsiteMiddleware will allow the request even if
-   its domain is not listed in allowed domains.
 
 RedirectMiddleware
 ------------------

docs/topics/exporters.rst

@@ -136,6 +136,70 @@ Example:
                   return f"$ {str(value)}"
               return super().serialize_field(field, name, value)
 
+.. _custom-exporters:
+
+Writing your own item exporter
+==============================
+
+To write an item exporter, subclass :class:`BaseItemExporter` and implement
+:meth:`~BaseItemExporter.export_item`, where
+:meth:`~BaseItemExporter.get_serialized_fields` gives you the ``(name, value)``
+pairs to export.
+
+To make your exporter available to the :ref:`feed exports
+<topics-feed-exports>`, list it in the :setting:`FEED_EXPORTERS` setting. Feed
+exports :ref:`build <from-crawler>` it with the output file as the first
+positional argument, and with the ``fields``, ``encoding`` and ``indent``
+:ref:`feed options <feed-options>` and every key of ``item_export_kwargs`` as
+keyword arguments, so your ``__init__`` method must forward unknown keyword
+arguments to :class:`BaseItemExporter`.
+
+The file object belongs to whoever opened it, i.e. to the feed storage in the
+case of feed exports, which also closes it. If you need a text file, for
+example to use :func:`csv.writer` or another Python API that does not accept a
+binary file, wrap it with :class:`io.TextIOWrapper` and call
+:meth:`~io.TextIOBase.detach` on the wrapper in
+:meth:`~BaseItemExporter.finish_exporting`; otherwise the wrapper closes the
+underlying file when it is garbage-collected.
+
+For example, the following item exporter writes items as blocks of
+``name: value`` lines:
+
+.. code-block:: python
+
+    from io import TextIOWrapper
+
+    from scrapy.exporters import BaseItemExporter
+
+
+    class TextItemExporter(BaseItemExporter):
+        def __init__(self, file, item_separator="\n", **kwargs):
+            super().__init__(**kwargs)
+            self.item_separator = item_separator
+            self.stream = TextIOWrapper(
+                file, encoding=self.encoding or "utf-8", write_through=True
+            )
+
+        def export_item(self, item):
+            for name, value in self.get_serialized_fields(item):
+                print(f"{name}: {value}", file=self.stream)
+            self.stream.write(self.item_separator)
+
+        def finish_exporting(self):
+            self.stream.detach()
+
+To use it as the ``txt`` feed format:
+
+.. code-block:: python
+
+    FEED_EXPORTERS = {"txt": "myproject.exporters.TextItemExporter"}
+    FEEDS = {
+        "items.txt": {
+            "format": "txt",
+            "item_export_kwargs": {"item_separator": "---\n"},
+        },
+    }
+
 .. _topics-exporters-reference:
 
 Built-in Item Exporters reference
@@ -168,6 +232,8 @@ BaseItemExporter
 
       Exports the given item. This method must be implemented in subclasses.
 
+   .. automethod:: BaseItemExporter.get_serialized_fields
+
    .. method:: serialize_field(field, name, value)
 
       Return the serialized value for the given field. You can override this

docs/topics/extensions.rst

@@ -374,8 +374,8 @@ This extension periodically logs rich stat data as a JSON object::
             "elapsed": 360.008903,
             "log_interval": 60.0,
             "log_interval_real": 60.006694,
-            "start_time": "2023-08-03 23:24:57",
+            "start_time": "2023-08-03T23:24:57.148903+00:00",
-            "utcnow": "2023-08-03 23:30:57"
+            "utcnow": "2023-08-03T23:30:57.157806+00:00"
         }
     }
 

docs/topics/feed-exports.rst

@@ -104,7 +104,8 @@ storage backend types which are defined by the URI scheme.
 The storages backends supported out of the box are:
 
 -   :ref:`topics-feed-storage-fs`
--   :ref:`topics-feed-storage-ftp`
+-   :ref:`feed-storage-ftp`
+-   :ref:`feed-storage-ftps`
 -   :ref:`topics-feed-storage-s3` (requires the :ref:`s3 <extras>` extra)
 -   :ref:`topics-feed-storage-gcs` (requires the :ref:`gcs <extras>` extra)
 -   :ref:`topics-feed-storage-stdout`
@@ -168,6 +169,7 @@ you specify a path (e.g. ``/tmp/export.csv``).
 Alternatively you can also use a :class:`pathlib.Path` object.
 
 .. _topics-feed-storage-ftp:
+.. _feed-storage-ftp:
 
 FTP
 ---
@@ -178,6 +180,9 @@ The feeds are stored in a FTP server.
 -   Example URI: ``ftp://user:pass@ftp.example.com/path/to/export.csv``
 -   Required external libraries: none
 
+FTP sends credentials and data in cleartext. Use :ref:`feed-storage-ftps`
+instead where possible.
+
 FTP supports two different connection modes: `active or passive
 <https://stackoverflow.com/a/1699163>`_. Scrapy uses the passive connection
 mode by default. To use the active connection mode instead, set the
@@ -192,6 +197,28 @@ storage backend is: ``True``.
 This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
 
 
+.. _feed-storage-ftps:
+
+FTPS
+----
+
+The feeds are stored in a FTP server, over a TLS connection, with the
+certificate of the server verified.
+
+.. versionadded:: VERSION
+
+-   URI scheme: ``ftps``
+-   Example URI: ``ftps://user:pass@ftp.example.com/path/to/export.csv``
+-   Required external libraries: none
+
+See :ref:`feed-storage-ftp` for connection modes, the ``overwrite`` default and
+file delivery.
+
+.. note:: For SFTP, an unrelated protocol built on SSH, use
+          `scrapy-feedexporter-sftp
+          <https://github.com/scrapy-plugins/scrapy-feedexporter-sftp>`_.
+
+
 .. _topics-feed-storage-s3:
 
 S3
@@ -502,7 +529,7 @@ as a fallback value if that key is not provided for a specific feed definition:
 
     -   :ref:`topics-feed-storage-fs`: ``False``
 
-    -   :ref:`topics-feed-storage-ftp`: ``True``
+    -   :ref:`feed-storage-ftp` and :ref:`feed-storage-ftps`: ``True``
 
         .. note:: Some FTP servers may not support appending to files (the
                   ``APPE`` FTP command).
@@ -624,6 +651,7 @@ Default:
         "s3": "scrapy.extensions.feedexport.S3FeedStorage",
         "gs": "scrapy.extensions.feedexport.GCSFeedStorage",
         "ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
+        "ftps": "scrapy.extensions.feedexport.FTPFeedStorage",
     }
 
 A dict containing the built-in feed storage backends supported by Scrapy. You

docs/topics/item-pipeline.rst

@@ -47,9 +47,17 @@ Additionally, they may also implement the following methods:
 
    This method is called when the spider is opened.
 
+   .. versionchanged:: VERSION
+      Added support for :exc:`~scrapy.exceptions.CloseSpider`.
+
+   It may raise :exc:`~scrapy.exceptions.CloseSpider` to close the spider before
+   it starts crawling, e.g. if a resource that the pipeline needs is
+   unavailable.
+
 .. method:: close_spider(self)
 
-   This method is called when the spider is closed.
+   This method is called when the spider is closed, before the
+   :signal:`spider_closed` signal is sent.
 
 Any of these methods may be defined as a coroutine function (``async def``).
 

docs/topics/media-pipeline.rst

@@ -178,6 +178,37 @@ By overriding ``file_path`` like this:
 
 For more information about the ``file_path`` method, see :ref:`topics-media-pipeline-override`.
 
+.. _file-naming-response:
+
+Naming files after the response
+-------------------------------
+
+``file_path`` also receives the ``response``, which allows naming files after
+response data. For example, to determine the file extension from the
+``Content-Type`` header, for URLs that do not end in a file name:
+
+.. code-block:: python
+
+    import mimetypes
+
+    from scrapy.pipelines.files import FilesPipeline
+
+
+    class ContentTypeFilesPipeline(FilesPipeline):
+        def file_path(self, request, response=None, info=None, *, item=None):
+            path = super().file_path(request, response, info, item=item)
+            if response is None:
+                return path
+            content_type = response.headers["Content-Type"].decode()
+            return path + (mimetypes.guess_extension(content_type) or "")
+
+This requires setting :setting:`FILES_EXPIRES` to ``0``. To find out whether a
+file has already been downloaded, Scrapy calls ``file_path`` before the
+download, with ``response`` set to ``None``, and checks the age of the file at
+the resulting path. A path that depends on the response can never match that
+check, and :setting:`FILES_EXPIRES` set to ``0`` disables it, at the cost of
+downloading every file on every run.
+
 .. _topics-supported-storage:
 
 Supported Storage
@@ -543,7 +574,7 @@ See here the methods that you can override in your custom Files Pipeline:
                 return "files/" + PurePosixPath(urlparse_cached(request).path).name
 
       Similarly, you can use the ``item`` to determine the file path based on some item
-      property.
+      property, or the ``response``, see :ref:`file-naming-response`.
 
       By default the :meth:`file_path` method returns
       ``full/<request URL hash>.<extension>``.
@@ -693,7 +724,7 @@ See here the methods that you can override in your custom Images Pipeline:
                 return "files/" + PurePosixPath(urlparse_cached(request).path).name
 
       Similarly, you can use the ``item`` to determine the file path based on some item
-      property.
+      property, or the ``response``, see :ref:`file-naming-response`.
 
       By default the :meth:`file_path` method returns
       ``full/<request URL hash>.<extension>``.

docs/topics/optimize.rst

@@ -0,0 +1,353 @@
+.. _optimize:
+
+============
+Optimization
+============
+
+A crawl goes as fast as its slowest part allows. :ref:`Find out which part that
+is <optimize-bottleneck>` before changing any setting.
+
+:ref:`Broad crawls <broad-crawls>` have their own set of recommended
+adjustments.
+
+.. _optimize-bottleneck:
+
+Finding the bottleneck
+======================
+
+The bottleneck depends on the spider: on the same machine, one crawl can be
+limited by its own parsing code and another by the target website. So measure
+the crawl that you want to optimize.
+
+:class:`~scrapy.extensions.logstats.LogStats` reports crawl speed every
+:setting:`LOGSTATS_INTERVAL` seconds:
+
+.. code-block:: text
+
+    [scrapy.extensions.logstats] INFO: Crawled 1200 pages (at 60 pages/min), scraped 1150 items (at 58 items/min)
+
+A rate that stays flat as you raise :setting:`CONCURRENT_REQUESTS` means
+something else is the limit.
+
+
+Reading the engine status
+-------------------------
+
+The :ref:`telnet console <topics-telnetconsole>` reports, through ``est()``,
+what every part of the engine is doing at a given moment:
+
+.. code-block:: text
+
+    len(engine.downloader.active)                   : 16
+    len(engine._slot.scheduler.mqs)                 : 92
+    len(engine.scraper.slot.active)                 : 0
+    engine.scraper.slot.active_size                 : 0
+    engine.scraper.slot.needs_backout()             : False
+
+Take a few readings at different points of the crawl:
+
+-   ``len(engine.downloader.active)`` stays at :setting:`CONCURRENT_REQUESTS`:
+    the downloader is the limit. You are waiting on the network or on the
+    target website. See :ref:`optimize-concurrency`.
+
+-   ``len(engine.downloader.active)`` stays below
+    :setting:`CONCURRENT_REQUESTS` while the scheduler queues (``mqs``,
+    ``dqs``) hold requests: something throttles those requests before they
+    reach the downloader, usually :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`,
+    :setting:`DOWNLOAD_DELAY` or :ref:`AutoThrottle <topics-autothrottle>`.
+
+-   Both the downloader and the scheduler queues stay near empty: your spider
+    is not producing requests fast enough. A crawl that walks pagination one
+    page at a time cannot use more concurrency than it creates. See
+    :ref:`optimize-requests`.
+
+-   ``needs_backout()`` is ``True``, or ``active_size`` approaches
+    :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE`: responses arrive faster than your
+    callbacks and :ref:`item pipelines <topics-item-pipeline>` handle them. The
+    bottleneck is your own code.
+
+-   ``len(engine._slot.scheduler.mqs)`` grows without settling: the crawl
+    discovers requests faster than it downloads them. This is what makes long
+    crawls run out of memory.
+
+
+Reading resource usage
+----------------------
+
+CPU
+    Scrapy runs in a single process, and everything except DNS resolution and
+    code you explicitly move to a thread runs in a single thread. One CPU core
+    is the ceiling; a process sitting at 100% of a core is CPU-bound no matter
+    how many cores the machine has.
+
+    Use a sampling profiler, such as py-spy_, to find out which code is
+    spending that CPU. :ref:`Selectors <topics-selectors>` and item pipelines
+    are the usual answer.
+
+    .. _py-spy: https://github.com/benfred/py-spy
+
+Memory
+    The :ref:`memory usage extension <topics-extensions-ref-memusage>` records
+    :stat:`memusage/startup` and :stat:`memusage/max`. A :stat:`memusage/max`
+    far above :stat:`memusage/startup` is expected; what matters is whether it
+    keeps growing for as long as the crawl runs.
+
+    Growth that tracks ``len(engine._slot.scheduler.mqs)`` is a scheduling
+    problem, covered in :ref:`optimize-memory`. Growth that does not is a
+    :ref:`memory leak <topics-leaks>`.
+
+Network
+    Compare :stat:`downloader/response_bytes` over the crawl time against your
+    available bandwidth. Saturated bandwidth caps concurrency regardless of any
+    setting.
+
+    DNS resolution is separate: it runs on a thread pool of
+    :setting:`REACTOR_THREADPOOL_MAXSIZE` threads, and results are cached
+    (:setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE`). It only becomes a
+    limit of its own when there are many different domains to resolve, as in
+    :ref:`broad crawls <broad-crawls>`, where it shows up as slow starts and
+    DNS timeouts.
+
+Disk
+    :ref:`Feed exports <topics-feed-exports>` write to disk on most crawls,
+    although item data is usually small enough for that not to matter. The ones
+    to suspect are
+    :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` and
+    the :ref:`media pipelines <topics-media-pipeline>`, which write whole
+    responses, and :setting:`JOBDIR`, which writes every scheduled request.
+
+
+.. _optimize-concurrency:
+
+Sending more requests at a time
+===============================
+
+:setting:`CONCURRENT_REQUESTS` caps how many requests are being downloaded at
+any given moment, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` caps how many of
+those may target the same domain, and :setting:`DOWNLOAD_DELAY` sets a minimum
+wait between two consecutive requests to the same domain. A project generated by
+:command:`startproject` gets one request per second per domain out of these.
+
+Raise them to crawl a single website faster, and see
+:ref:`broad-crawls-concurrency` to spread requests across many websites
+instead.
+
+The limit that matters, though, is the one the target website tolerates.
+Exceeding it gets you throttled, served errors or banned, all of which make the
+crawl slower than a lower concurrency would have been. To find that limit:
+
+-   Read the :ref:`robots.txt <topics-dlmw-robots>` file of the website. Scrapy
+    does not act on its ``Crawl-delay`` and ``Request-rate`` directives, so when
+    they are present, translate them into :setting:`DOWNLOAD_DELAY` and
+    concurrency settings yourself.
+
+-   Check the traffic that the website already gets, using a service like
+    `SimilarWeb`_ or `Cloudflare Radar`_. A rate that is a rounding error next
+    to what the website serves anyway is unlikely to be a problem for it.
+
+    .. _SimilarWeb: https://www.similarweb.com/
+    .. _Cloudflare Radar: https://radar.cloudflare.com/
+
+-   Look for a documented way in. An API, a bulk export or a search endpoint is
+    both faster for you and cheaper for the website than crawling its pages, and
+    the terms of service may state a rate.
+
+-   Crawl when the website is idle, in its own timezone, so that the capacity
+    you take is capacity nobody else wanted.
+
+-   Raise concurrency gradually and watch the website respond.
+    :stat:`downloader/response_status_count/{status_code}` counts for 429, 503
+    or the ban page of the website, growing :stat:`retry/count`, or a
+    :ref:`download latency <download-latency>` that climbs as you push harder,
+    all mean you have gone past the limit.
+
+
+.. _optimize-requests:
+
+Producing requests faster
+=========================
+
+A spider that discovers its requests one response at a time keeps the
+downloader idle no matter how high you set :setting:`CONCURRENT_REQUESTS`. To
+put more requests in the scheduler earlier:
+
+-   Request every page at once when you can work out how many there are, e.g.
+    from a page count or from a result count and a page size in the first
+    response, instead of following a link to the next page on every response.
+
+-   Get URLs from a source that lists many of them at once, such as a sitemap
+    or a search or export endpoint of the target website. For a crawl that
+    needs nothing else, :class:`~scrapy.spiders.SitemapSpider` reads sitemaps
+    for you.
+
+-   Raise the :attr:`~scrapy.Request.priority` of pagination requests, so that
+    they are downloaded before the requests that they compete with, and
+    discover the rest of the crawl sooner.
+
+Each of these trades memory for speed: a request produced before the downloader
+can take it waits in the scheduler, or on disk if you set :setting:`JOBDIR`.
+Pushed far enough, they turn memory or disk into your new bottleneck, which is
+why :ref:`optimize-memory` recommends the reverse of the last point.
+
+
+.. _optimize-resources:
+
+Lowering resource usage
+=======================
+
+.. _optimize-memory:
+
+Lowering memory usage
+---------------------
+
+-   Lower :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE`.
+
+-   Lower :setting:`DOWNLOAD_MAXSIZE`, which allows a single response to take up
+    to 1 GiB of memory by default, multiplied by your concurrency. Set
+    :setting:`DOWNLOAD_WARNSIZE` first to find out whether the website actually
+    serves responses that big.
+
+-   Lower the number of :ref:`scheduled requests <topics-scheduler>` held in
+    memory:
+
+    -   Increase the :attr:`~scrapy.Request.priority` of requests whose
+        :attr:`~scrapy.Request.callback` cannot yield additional requests.
+
+        For example, the following spider uses a higher priority (1) for book
+        requests than for pagination requests:
+
+        .. code-block:: python
+
+            from scrapy import Spider
+
+
+            class BooksToScrapeComSpider(Spider):
+                name = "books_toscrape_com"
+                start_urls = [
+                    "http://books.toscrape.com/catalogue/category/books/mystery_3/index.html"
+                ]
+
+                def parse(self, response):
+                    next_page_links = response.css(".next a")
+                    yield from response.follow_all(next_page_links)
+                    book_links = response.css("article a")
+                    yield from response.follow_all(book_links, callback=self.parse_book, priority=1)
+
+                def parse_book(self, response):
+                    yield {
+                        "name": response.css("h1::text").get(),
+                        "price": response.css(".price_color::text").re_first("£(.*)"),
+                        "url": response.url,
+                    }
+
+        .. note:: If the number of request-yielding, low-priority requests
+            scheduled at any given time is lower than concurrency settings
+            (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
+            :setting:`CONCURRENT_REQUESTS`), as in the example above, this can
+            slow down your crawl by turning those requests into a bottleneck.
+
+    -   If you have many :ref:`start requests <start-requests>`, consider
+        :ref:`delaying their iteration <start-requests-lazy>`.
+
+    -   Set :setting:`JOBDIR` to offload all scheduled requests to disk.
+
+-   Be on the lookout for :ref:`memory leaks <topics-leaks>`.
+
+
+Lowering network usage
+----------------------
+
+-   Install brotli_ and zstandard_ to support brotli-compressed_ and
+    zstd-compressed_ responses.
+
+    .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
+    .. _brotli: https://pypi.org/project/Brotli/
+    .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
+    .. _zstandard: https://pypi.org/project/zstandard/
+
+-   Enable :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
+    while developing your spider, so that re-runs do not download the same
+    responses again.
+
+
+Lowering CPU usage
+------------------
+
+-   Set :setting:`LOG_LEVEL` to ``"INFO"`` or higher.
+
+-   Restrict what you parse. A :ref:`selector <topics-selectors>` over a
+    smaller part of the response, or a single query whose result you reuse,
+    beats repeated queries over the whole document.
+
+
+Other tips
+----------
+
+-   Try :ref:`using the asyncio reactor <install-asyncio>` with uvloop_ as
+    :ref:`custom event loop <using-custom-loops>`, i.e. setting
+    :setting:`ASYNCIO_EVENT_LOOP` to ``"uvloop.Loop"``.
+
+    .. _uvloop: https://github.com/MagicStack/uvloop
+
+    Alternatively, try :ref:`switching to a non-asyncio reactor
+    <disable-asyncio>`.
+
+-   Disable unused :ref:`components <topics-components>`.
+
+    For example, set :setting:`COOKIES_ENABLED` to ``False`` unless you need
+    cookies.
+
+-   Split the crawl across separate processes to use more than one CPU core.
+    See :ref:`distributed-crawls`.
+
+
+.. _broad-crawls:
+.. _topics-broad-crawls:
+
+Speeding up broad crawls
+========================
+
+While Scrapy is well suited for **broad crawls**, i.e. crawls that target many
+websites, the default :ref:`settings <topics-settings>` are optimized for
+crawls targeting a single website.
+
+For broad crawls, consider these adjustments:
+
+-   .. _broad-crawls-concurrency:
+
+    Increase the global concurrency:
+
+    -   Set :setting:`CONCURRENT_REQUESTS` as close to
+        :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` × [number of target domains]
+        (e.g. 8 × 10 domains = 80 concurrent requests) as your CPU and memory
+        allow.
+
+    -   Increase :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` when increasing
+        :setting:`CONCURRENT_REQUESTS` stops making a difference.
+
+-   .. _broad-crawls-bfo:
+
+    If memory is a bottleneck, see if :ref:`crawling in BFO order <bfo>` lowers
+    memory usage.
+
+-   Improve DNS resolution speed:
+
+    -   Set up your own DNS server, with a local cache and upstream to a `large
+        DNS server`_, to avoid slowing down your network.
+
+        .. _large DNS server: https://en.wikipedia.org/wiki/Public_recursive_name_server#Notable_public_DNS_service_operators
+
+    -   Increase :setting:`REACTOR_THREADPOOL_MAXSIZE` to the minimum value
+        that avoids DNS resolution timeouts and makes a noticeable positive
+        impact in crawl speed.
+
+-   Lower the negative impact of some responses:
+
+    -   Set :setting:`RETRY_ENABLED` to ``False`` or, if you need retries,
+        consider lowering :setting:`RETRY_TIMES`.
+
+    -   Lower :setting:`DOWNLOAD_TIMEOUT` to a more reasonable value, to
+        discard stuck requests more quickly.
+
+    -   Set :setting:`REDIRECT_ENABLED` to ``False`` unless you want to follow
+        redirects.

docs/topics/practices.rst

@@ -458,6 +458,18 @@ finishes before starting the next one:
     should not have a different value per spider, and :ref:`pre-crawler
     settings <pre-crawler-settings>` cannot be defined per spider.
 
+Every other setting applies to each crawler separately. This includes
+concurrency and politeness settings, such as :setting:`CONCURRENT_REQUESTS`,
+:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and :setting:`DOWNLOAD_DELAY`, and
+:ref:`AutoThrottle <topics-autothrottle>` also throttles each crawler
+separately. When crawling simultaneously, divide those values by the number of
+crawlers to keep the combined load on your hardware and on target websites
+unchanged.
+
+Because of this, running the same spider several times in the same process
+multiplies those limits instead of increasing crawling capacity. To crawl
+faster, raise :setting:`CONCURRENT_REQUESTS` on a single crawler.
+
 .. seealso:: :ref:`run-from-script`.
 
 .. skip: end
@@ -518,32 +530,41 @@ modules by separating them with commas.
 Avoiding getting banned
 =======================
 
-Some websites implement certain measures to prevent bots from crawling them,
+Websites tell regular visitors and crawlers apart by how their traffic looks:
-with varying degrees of sophistication. Getting around those measures can be
+the headers it carries, how fast it arrives, how many requests come from the
-difficult and tricky, and may sometimes require special infrastructure. Please
+same place. Traffic that stands out can be blocked even when the crawling
-consider contacting `commercial support`_ if in doubt.
+itself would be welcome.
 
-Here are some tips to keep in mind when dealing with these kinds of sites:
+Where the website allows crawling, the most effective thing you can do is make
+yourself known: set :setting:`USER_AGENT` to a value that identifies you and
+lets its owners reach you, so that they can ask you to adjust your crawler
+rather than block it.
 
-* rotate your user agent from a pool of well-known ones from browsers (Google
+Where that is not enough, the following make your traffic resemble that of a
-  around to get a list of them)
+regular visitor:
-* disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use
+
-  cookies to spot bot behaviour
+* rotate your user agent among those of common browsers, so that your requests
-* use download delays (2 or higher). See :setting:`DOWNLOAD_DELAY` setting.
+  do not all look alike (search the web for an up-to-date list)
-* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
+* disable cookies (see :setting:`COOKIES_ENABLED`), so that a session
-  directly
+  identifier does not tie all your requests together
-* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
+* space out your requests, 2 seconds apart or more, with the
+  :setting:`DOWNLOAD_DELAY` setting, to keep your pace closer to that of a
+  person browsing
+* where possible, read pages from `Common Crawl`_, which sends no traffic to
+  the website at all
+* spread your requests over a pool of IP addresses, so that none of them
+  accounts for your whole crawl. For example, the free `Tor project`_ or paid
   services like `ProxyMesh`_.
-* for HTTPS websites, if blocking appears related to TLS behavior, consider
+* match the TLS behavior of a browser: some websites respond differently
-  adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and
+  depending on the TLS version of the client, which you can adjust with the
-  :setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond
+  :setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION`
-  differently depending on the TLS method used by the client.
+  settings.
-* use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy
+* let a service take care of all of the above, such as `Zyte API`_, which
-  plugin <https://github.com/scrapy-plugins/scrapy-zyte-api>`__ and additional
+  provides a `Scrapy plugin
+  <https://github.com/scrapy-plugins/scrapy-zyte-api>`__ and additional
   features, like `AI web scraping <https://www.zyte.com/ai-web-scraping/>`__
 
-If you are still unable to prevent your bot getting banned, consider contacting
+If your crawler still gets blocked, consider contacting `commercial support`_.
-`commercial support`_.
 
 .. _static-analysis:
 

docs/topics/request-response.rst

@@ -53,65 +53,13 @@ Request objects
        ``None`` is passed as value, the HTTP header will not be sent at all.
 
        .. caution:: Cookies set via the ``Cookie`` header are not considered by the
-           :ref:`cookies-mw`. If you need to set cookies for a request, use the
+           :ref:`cookie middleware <cookies>`. If you need to set cookies for a
-           ``cookies`` argument. This is a known current limitation that is being
+           request, use the ``cookies`` argument.
-           worked on.
 
     :type headers: dict
 
-    :param cookies: the request cookies. These can be sent in two forms.
+    :param cookies: the request cookies, as a dict of cookie names and values
-
+        or as a list of dicts with a cookie each. See :ref:`cookies`.
-        .. invisible-code-block: python
-
-            from scrapy import Request
-
-        1. Using a dict:
-
-        .. code-block:: python
-
-            request_with_cookies = Request(
-                url="http://www.example.com",
-                cookies={"currency": "USD", "country": "UY"},
-            )
-
-        2. Using a list of dicts:
-
-        .. code-block:: python
-
-            request_with_cookies = Request(
-                url="https://www.example.com",
-                cookies=[
-                    {
-                        "name": "currency",
-                        "value": "USD",
-                        "domain": "example.com",
-                        "path": "/currency",
-                        "secure": True,
-                    },
-                ],
-            )
-
-        The latter form allows for customizing the ``domain`` and ``path``
-        attributes of the cookie. This is only useful if the cookies are saved
-        for later requests.
-
-        .. reqmeta:: dont_merge_cookies
-
-        When some site returns cookies (in a response) those are stored in the
-        cookies for that domain and will be sent again in future requests.
-        That's the typical behaviour of any regular web browser.
-
-        Note that setting the :reqmeta:`dont_merge_cookies` key to ``True`` in
-        :attr:`request.meta <scrapy.Request.meta>` causes custom cookies to be
-        ignored.
-
-        For more info see :ref:`cookies-mw`.
-
-        .. caution:: Cookies set via the ``Cookie`` header are not considered by the
-            :ref:`cookies-mw`. If you need to set cookies for a request, use the
-            :class:`scrapy.Request.cookies <scrapy.Request>` parameter. This is a known
-            current limitation that is being worked on.
-
     :type cookies: dict or list
 
     :param encoding: the encoding of this request (defaults to ``'utf-8'``).
@@ -770,6 +718,10 @@ is raise while processing it.
 It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
 be used to track connection establishment timeouts, DNS errors etc.
 
+If an errback raises an exception, Scrapy logs it and sends the
+:signal:`spider_error` signal, unless the exception is the one that the errback
+received, which Scrapy logs as a download error instead.
+
 Here's an example spider logging all errors and catching some specific
 errors if needed:
 
@@ -1428,9 +1380,6 @@ TextResponse objects
 
     .. automethod:: TextResponse.json()
 
-        Returns a Python object from deserialized JSON document.
-        The result is cached after the first call.
-
     .. method:: TextResponse.urljoin(url)
 
         Constructs an absolute url by combining the Response's base url with

docs/topics/security.rst

@@ -36,6 +36,77 @@ their input in an unsafe way, such as :func:`eval`, :func:`exec`, or
 :func:`pickle.loads`, and be careful when writing response data to paths
 derived from the response itself.
 
+.. _security-response-size:
+
+Memory use when parsing responses
+=================================
+
+Parsing a response with :ref:`selectors <topics-selectors>` builds an in-memory
+tree of the whole response body, which takes several times as much memory as
+the body itself. Scrapy parses without the size limits that libxml2 applies by
+default, so the size of that tree is bound only by the size of the response, as
+controlled by :setting:`DOWNLOAD_MAXSIZE` (default: 1 GiB).
+
+XML entities are left unresolved, so the tree stays proportional to the
+response body even for input crafted as an `XML bomb
+<https://lxml.de/FAQ.html#is-lxml-vulnerable-to-xml-bombs>`_. A server can still
+make a crawler allocate a lot of memory by returning a very large response,
+though, so if you know the size of the responses you care about, lower the
+limit:
+
+.. code-block:: python
+
+    DOWNLOAD_MAXSIZE = 32 * 1024 * 1024  # 32 MiB
+
+* **Pro:** a server cannot make the crawler allocate more memory than the limit
+  allows, whether by returning a large response or by crafting one that is
+  expensive to parse.
+
+* **Con:** you can no longer scrape sites that legitimately serve responses
+  above the limit, as those responses are dropped.
+
+.. _security-parser-limits:
+
+Parser limits
+-------------
+
+The limits that libxml2 applies by default, such as 256 nesting levels and
+10 MB per text node, can be restored by overriding
+:attr:`~scrapy.http.TextResponse.selector` in a response subclass and swapping
+responses in a :ref:`downloader middleware <topics-downloader-middleware>`:
+
+.. code-block:: python
+
+    from functools import cached_property
+
+    from scrapy import Selector
+    from scrapy.http import HtmlResponse
+
+
+    class LimitedHtmlResponse(HtmlResponse):
+        @cached_property
+        def selector(self):
+            return Selector(self, huge_tree=False)
+
+
+    class LimitedParsingMiddleware:
+        def process_response(self, request, response, spider):
+            if isinstance(response, HtmlResponse):
+                return response.replace(cls=LimitedHtmlResponse)
+            return response
+
+Do the same with :class:`~scrapy.http.XmlResponse` if you also parse XML.
+
+These limits apply per node, so :setting:`DOWNLOAD_MAXSIZE` remains your bound
+on total memory: a response made of many small elements is parsed in full and
+uses as much memory either way.
+
+* **Pro:** deeply nested responses, and responses with very large individual
+  nodes, become cheaper to parse.
+
+* **Con:** parsing stops at those limits without raising, so a legitimate page
+  that exceeds them yields incomplete data and no error.
+
 TLS connections
 ===============
 

docs/topics/settings.rst

@@ -69,9 +69,10 @@ Example::
 precedence and override the project ones.
 
 .. note:: :ref:`Pre-crawler settings <pre-crawler-settings>` cannot be defined
-    per spider, and :ref:`reactor settings <reactor-settings>` should not have
+    per spider, and :ref:`reactor settings <reactor-settings>` and
-    a different value per spider when :ref:`running multiple spiders in the
+    :ref:`logging settings <logging-settings>` are subject to restrictions when
-    same process <run-multiple-spiders>`.
+    :ref:`running multiple spiders in the same process
+    <run-multiple-spiders>`.
 
 One way to do so is by setting their :attr:`~scrapy.Spider.custom_settings`
 attribute:
@@ -329,32 +330,41 @@ Reactor settings
 **Reactor settings** are settings tied to the :doc:`Twisted reactor
 <twisted:core/howto/reactor-basics>`.
 
-These settings can be defined from a spider. However, because only 1 reactor
+Because only 1 reactor can be used per process, these settings cannot use a
-can be used per process, these settings cannot use a different value per spider
+different value per spider when :ref:`running multiple spiders in the same
-when :ref:`running multiple spiders in the same process
+process <run-multiple-spiders>`.
-<run-multiple-spiders>`.
 
-In general, if different spiders define different values, the first defined
+These settings are used upon installing the reactor:
-value is used. However, if two spiders request a different reactor, an
-exception is raised.
-
-These settings are:
 
 -   :setting:`ASYNCIO_EVENT_LOOP` (not possible to set per-spider when using
     :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below)
 
+-   :setting:`TWISTED_REACTOR` (ignored when using
+    :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below)
+
+They can be :ref:`set from a spider <spider-settings>`, but only the values
+from the first spider that runs are used, since that is when the reactor is
+installed. If a later spider asks for a different reactor or a different event
+loop, an exception is raised. With
+:class:`~scrapy.crawler.CrawlerRunner` and
+:class:`~scrapy.crawler.AsyncCrawlerRunner` the reactor must be installed
+beforehand, so these settings are only used to check that the installed reactor
+and event loop match them.
+
+These settings are applied when starting the reactor:
+
 -   :setting:`TWISTED_DNS_RESOLVER` and settings used by the corresponding
     component, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE`
     and :setting:`DNS_TIMEOUT` for the default one.
 
 -   :setting:`REACTOR_THREADPOOL_MAXSIZE`
 
--   :setting:`TWISTED_REACTOR` (ignored when using
+They are read from the settings of the
-    :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below)
+:class:`~scrapy.crawler.CrawlerProcess` or
-
+:class:`~scrapy.crawler.AsyncCrawlerProcess` object, so setting them from a
-:setting:`ASYNCIO_EVENT_LOOP` and :setting:`TWISTED_REACTOR` are used upon
+spider or an :ref:`add-on <topics-addons>` has no effect. They are ignored
-installing the reactor. The rest of the settings are applied when starting
+altogether when using :class:`~scrapy.crawler.CrawlerRunner` or
-the reactor.
+:class:`~scrapy.crawler.AsyncCrawlerRunner`, which do not start the reactor.
 
 There is an additional restriction for :setting:`TWISTED_REACTOR` and
 :setting:`ASYNCIO_EVENT_LOOP` when using
@@ -654,9 +664,13 @@ The default headers used for Scrapy HTTP Requests. They're populated in the
 :class:`~scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware`.
 
 .. caution:: Cookies set via the ``Cookie`` header are not considered by the
-    :ref:`cookies-mw`. If you need to set cookies for a request, use the
+    :ref:`cookie middleware <cookies>`. If you need to set cookies for a
-    :class:`Request.cookies <scrapy.Request>` parameter. This is a known
+    request, use the :class:`Request.cookies <scrapy.Request>` parameter.
-    current limitation that is being worked on.
+
+.. caution:: A ``Referer`` header defined here only reaches requests for which
+    :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` does not set
+    one, such as start requests. To send it on every request, set
+    :setting:`REFERRER_POLICY` to ``"no-referrer"``.
 
 .. setting:: DEPTH_LIMIT
 
@@ -749,6 +763,11 @@ Default: ``60``
 
 Timeout for processing of DNS queries in seconds. Float is supported.
 
+The timeout starts when the query is queued into the Twisted reactor thread
+pool, not when it is sent. If that thread pool is saturated, queries can time
+out before being sent, in which case increasing
+:setting:`REACTOR_THREADPOOL_MAXSIZE` helps more than increasing this setting.
+
 .. note::
     This setting is only used by
     :class:`~scrapy.resolver.CachingThreadedResolver`. It has no effect when
@@ -1374,7 +1393,7 @@ FEED_TEMPDIR
 Default: ``None``
 
 The Feed Temp dir allows you to set a custom folder to save crawler
-temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
+temporary files before uploading with :ref:`FTP feed storage <feed-storage-ftp>` and
 :ref:`Amazon S3 <topics-feed-storage-s3>`.
 
 .. setting:: FEED_STORAGE_GCS_ACL
@@ -1903,6 +1922,7 @@ Type of in-memory queue used by the scheduler. Other available type is:
 
 
 .. setting:: SCHEDULER_PRIORITY_QUEUE
+.. _broad-crawls-scheduler-priority-queue:
 
 SCHEDULER_PRIORITY_QUEUE
 ------------------------
@@ -2328,6 +2348,11 @@ also used by :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware
 if :setting:`ROBOTSTXT_USER_AGENT` setting is ``None`` and
 there is no overriding User-Agent header specified for the request.
 
+Set it to a value that identifies you, including a URL or an email address
+where website owners can reach you, e.g. ``"MyProject
+(+https://example.com/bot)"``, so that they can ask you to adjust your crawler
+rather than block it.
+
 .. setting:: WARN_ON_GENERATOR_RETURN_VALUE
 
 WARN_ON_GENERATOR_RETURN_VALUE

docs/topics/shell.rst

@@ -144,6 +144,32 @@ Those objects are:
 
 -   ``settings`` - the current :ref:`Scrapy settings <topics-settings>`
 
+.. _shell-update-vars:
+
+Adding your own objects
+-----------------------
+
+To define additional objects, or to run code every time a response is fetched,
+write a :ref:`custom project command <topics-commands>` in a module called
+``shell``, which overrides the :command:`shell` command, and override its
+``update_vars`` method. It is called on start and after every ``fetch``, and it
+receives the mapping of variable names to objects:
+
+.. code-block:: python
+
+    from scrapy.commands.shell import Command as ShellCommand
+
+
+    class Command(ShellCommand):
+        def update_vars(self, vars):
+            from myproject.utils import parse_product
+
+            vars["parse_product"] = parse_product
+            if vars["response"] is not None:
+                vars["product"] = parse_product(vars["response"])
+
+``response`` is ``None`` when the shell is started without a URL.
+
 Example of shell session
 ========================
 

docs/topics/signals.rst

@@ -44,6 +44,15 @@ Here is a simple example showing how you can catch signals and perform some acti
         def parse(self, response):
             pass
 
+.. _signal-order:
+
+Handler order
+=============
+
+The order in which the handlers of a signal run is undefined, and
+:ref:`asynchronous handlers <signal-deferred>` run concurrently. If two actions
+must happen in a given order, run both from a single handler, in that order.
+
 .. _signal-deferred:
 
 Asynchronous signal handlers
@@ -149,6 +158,15 @@ scheduler_empty
 
     See :ref:`start-requests-lazy` for an example.
 
+    .. warning:: Only wait for this signal from
+        :meth:`~scrapy.Spider.start`. While no request can be sent, e.g. while
+        the responses being parsed exceed
+        :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE`, the engine does not ask the
+        scheduler for requests, and hence this signal is not sent. So waiting
+        for it from a :ref:`callback <callbacks>` can hang the crawl,
+        because the response being parsed is itself one of the responses that
+        may be blocking requests.
+
     This signal does not support :ref:`asynchronous handlers <signal-deferred>`.
 
 
@@ -272,6 +290,13 @@ spider_opened
     reserve per-spider resources, but can be used for any task that needs to be
     performed when a spider is opened.
 
+    .. versionchanged:: VERSION
+       Added support for :exc:`~scrapy.exceptions.CloseSpider`.
+
+    You may raise a :exc:`~scrapy.exceptions.CloseSpider` exception to close the
+    spider before it starts crawling, e.g. if a resource that the spider needs
+    is unavailable.
+
     This signal supports :ref:`asynchronous handlers <signal-deferred>`.
 
     :param spider: the spider which has been opened
@@ -320,15 +345,22 @@ spider_error
 .. signal:: spider_error
 .. function:: spider_error(failure, response, spider)
 
-    Sent when a spider callback generates an error (i.e. raises an exception).
+    Sent when a spider callback or the :meth:`~scrapy.Spider.start` method of a
+    spider generates an error (i.e. raises an exception).
+
+    .. versionchanged:: VERSION
+       Exceptions from :meth:`~scrapy.Spider.start` are also reported, see
+       :ref:`start-error`.
 
     This signal does not support :ref:`asynchronous handlers <signal-deferred>`.
 
     :param failure: the exception raised
     :type failure: twisted.python.failure.Failure
 
-    :param response: the response being processed when the exception was raised
+    :param response: the response being processed when the exception was
-    :type response: :class:`~scrapy.http.Response` object
+        raised, or ``None`` if the exception came from
+        :meth:`~scrapy.Spider.start`.
+    :type response: :class:`~scrapy.http.Response` | ``None``
 
     :param spider: the spider which raised the exception
     :type spider: :class:`~scrapy.Spider` object

docs/topics/spider-middleware.rst

@@ -122,6 +122,9 @@ one or more of these methods:
         This method is an :term:`asynchronous generator` called with the
         results from the spider after the spider has processed the response.
 
+        *result* is lazy: a generator callback runs as *result* is iterated, so
+        code that runs before that iteration runs before the callback body.
+
         .. seealso:: :ref:`universal-spider-middleware`.
 
         :param response: the response which generated this output from the
@@ -142,8 +145,9 @@ one or more of these methods:
 
     .. method:: process_spider_exception(response, exception)
 
-        This method is called when a spider or :meth:`process_spider_output`
+        This method is called when a spider callback or a
-        method (from a previous spider middleware) raises an exception.
+        :meth:`process_spider_output` method (from a previous spider
+        middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
         iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
@@ -224,25 +228,7 @@ DepthMiddleware
 .. module:: scrapy.spidermiddlewares.depth
    :synopsis: Depth Spider Middleware
 
-.. class:: DepthMiddleware
+.. autoclass:: DepthMiddleware
-
-   DepthMiddleware is used for tracking the depth of each Request inside the
-   site being scraped. It works by setting ``request.meta['depth'] = 0`` whenever
-   there is no value previously set (usually just the first Request) and
-   incrementing it by 1 otherwise.
-
-   It can be used to limit the maximum depth to scrape, control Request
-   priority based on their depth, and things like that.
-
-   The :class:`DepthMiddleware` can be configured through the following
-   settings (see the settings documentation for more info):
-
-      * :setting:`DEPTH_LIMIT` - The maximum depth that will be allowed to
-        crawl for any site. If zero, no limit will be imposed.
-      * :setting:`DEPTH_STATS_VERBOSE` - Whether to collect the number of
-        requests for each depth.
-      * :setting:`DEPTH_PRIORITY` - Whether to prioritize the requests based on
-        their depth.
 
 HttpErrorMiddleware
 -------------------

docs/topics/spiders.rst

@@ -59,9 +59,16 @@ scrapy.Spider
        :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` is
        enabled.
 
+       .. versionchanged:: VERSION
+          Changes to this attribute during a crawl are now taken into account.
+
        Let's say your target url is ``https://www.example.com/1.html``,
        then add ``'example.com'`` to the list.
 
+       You may modify this attribute while the spider runs, e.g. to allow
+       domains that you only learn about from an earlier response. The change
+       affects requests scheduled after it.
+
    .. autoattribute:: start_urls
 
    .. attribute:: custom_settings
@@ -389,8 +396,12 @@ Start requests
 Delaying start request iteration
 --------------------------------
 
-You can override the :meth:`~scrapy.Spider.start` method as follows to pause
+Scrapy iterates :meth:`~scrapy.Spider.start` as fast as it yields, so all start
-its iteration whenever there are scheduled requests:
+requests reach the scheduler early in the crawl, however many they are. To
+minimize the number of requests in the scheduler at any given time, and with it
+resource usage (memory, or disk when using :setting:`JOBDIR`), override
+:meth:`~scrapy.Spider.start` to pause its iteration whenever there are
+scheduled requests:
 
 .. code-block:: python
 
@@ -400,9 +411,37 @@ its iteration whenever there are scheduled requests:
                 await self.crawler.signals.wait_for(signals.scheduler_empty)
             yield item_or_request
 
-This can help minimize the number of requests in the scheduler at any given
+.. _start-error:
-time, to minimize resource usage (memory or disk, depending on
+
-:setting:`JOBDIR`).
+Handling start errors
+---------------------
+
+An exception raised by :meth:`~scrapy.Spider.start` ends its iteration, so any
+remaining start items and requests are never sent. Scrapy logs the exception,
+sends the :signal:`spider_error` signal, and, once the already scheduled
+requests are done, closes the spider with the ``start_error``
+:stat:`finish_reason`.
+
+.. versionchanged:: VERSION
+   The close reason used to be ``finished``, and neither the
+   :signal:`spider_error` signal nor the :stat:`spider_exceptions/count` stat
+   reported the exception.
+
+To keep the iteration going, catch the exception yourself:
+
+.. code-block:: python
+
+    async def start(self):
+        for url in self.start_urls:
+            try:
+                request = Request(url)
+            except ValueError:
+                self.logger.exception(f"Skipping start URL {url}")
+            else:
+                yield request
+
+To stop the crawl instead, and choose your own :stat:`finish_reason`, raise
+:exc:`~scrapy.exceptions.CloseSpider`.
 
 .. _builtin-spiders:
 

docs/topics/stats.rst

@@ -121,6 +121,13 @@ one per actual value of the placeholder.
     :meth:`~scrapy.statscollectors.StatsCollector.get_stats` output is
     equivalent to a counter of 0.
 
+.. stat:: depth/request_ignored_count
+
+``depth/request_ignored_count``
+    Number of requests dropped for exceeding :setting:`DEPTH_LIMIT`.
+
+    Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`.
+
 .. stat:: downloader/exception_count
 
 ``downloader/exception_count``
@@ -294,6 +301,10 @@ one per actual value of the placeholder.
     -   ``shutdown``: the crawl was interrupted, e.g. by a system signal such
         as ``SIGINT`` (:kbd:`Ctrl-C`).
 
+    -   ``start_error``: :meth:`~scrapy.Spider.start` raised an exception, so
+        some :ref:`start requests <start-requests>` may never have been sent,
+        see :ref:`start-error`.
+
     Third-party components and your own code may use any other reason, e.g. by
     raising :exc:`~scrapy.exceptions.CloseSpider` with it.
 
@@ -724,18 +735,21 @@ one per actual value of the placeholder.
 .. stat:: spider_exceptions/count
 
 ``spider_exceptions/count``
-    Number of unhandled exceptions raised by spider callbacks.
+    Number of unhandled exceptions raised by spider callbacks or by
+    :meth:`~scrapy.Spider.start`.
 
-    Set by the :ref:`scraper <topics-architecture>`.
+    Set by the :ref:`engine <topics-architecture>` and the :ref:`scraper
+    <topics-architecture>`.
 
 .. stat:: spider_exceptions/{exception}
 
 ``spider_exceptions/{exception}``
-    Number of unhandled exceptions raised by spider callbacks, per exception,
+    Same as :stat:`spider_exceptions/count`, per exception, where
-    where ``{exception}`` is the class name of the exception, e.g.
+    ``{exception}`` is the class name of the exception, e.g.
     ``spider_exceptions/ValueError``.
 
-    Set by the :ref:`scraper <topics-architecture>`.
+    Set by the :ref:`engine <topics-architecture>` and the :ref:`scraper
+    <topics-architecture>`.
 
 .. stat:: start_time
 

pyproject.toml

@@ -26,6 +26,8 @@ dependencies = [
     # Platform-specific dependencies
     'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',
     'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"',
+    'brotli>=1.2.0; implementation_name != "pypy"',
+    'brotlicffi>=1.2.0.0; implementation_name == "pypy"',
 ]
 classifiers = [
     "Development Status :: 5 - Production/Stable",
@@ -62,10 +64,6 @@ Tracker = "https://github.com/scrapy/scrapy/issues"
 
 [project.optional-dependencies]
 bpython = ["bpython>=0.7.1"]
-brotli = [
-    "brotli>=1.2.0; implementation_name != 'pypy'",
-    "brotlicffi>=1.2.0.0; implementation_name == 'pypy'",
-]
 gcs = ["google-cloud-storage>=1.29.0"]
 httpx = ["httpx2[http2,socks]>=2.0.0"]
 images = ["Pillow>=8.3.2"]
@@ -125,23 +123,11 @@ module = [
     "tests.test_downloaderslotssettings",
     "tests.test_dupefilters",
     "tests.test_engine_loop",
-    "tests.test_exporters",
     "tests.test_extension_statsmailer",
     "tests.test_extension_throttle",
-    "tests.test_feedexport",
-    "tests.test_feedexport_postprocess",
-    "tests.test_feedexport_storages",
-    "tests.test_feedexport_uri_params",
-    "tests.test_item",
     "tests.test_linkextractors",
-    "tests.test_loader",
     "tests.test_logformatter",
     "tests.test_mail",
-    "tests.test_pipeline_crawl",
-    "tests.test_pipeline_files",
-    "tests.test_pipeline_images",
-    "tests.test_pipeline_media",
-    "tests.test_pipelines",
     "tests.test_pqueues",
     "tests.test_scheduler_base",
     "tests.test_settings",
@@ -333,6 +319,9 @@ markers = [
 ]
 filterwarnings = [
     "ignore::DeprecationWarning:twisted.web.static",
+    # Jobs that do not report coverage disable it with --no-cov, which pytest-cov
+    # warns about because the coverage options below stay in place.
+    "ignore::pytest_cov.CovDisabledWarning",
     # Twisted doesn't close failed sockets after CannotListenError: https://github.com/twisted/twisted/issues/6108
     "ignore:Exception ignored in. <socket\\.socket.*laddr=..0\\.0\\.0\\.0., 0.:pytest.PytestUnraisableExceptionWarning",
 ]

scrapy/commands/parse.py

@@ -281,7 +281,6 @@ class Command(BaseRunSpiderCommand):
     ) -> list[Any]:
         items, requests, opts, depth, spider, callback = args
         if opts.pipelines:
-            assert self.pcrawler.engine
             itemproc = self.pcrawler.engine.scraper.itemproc
             if hasattr(itemproc, "process_item_async"):
                 for item in items:

scrapy/commands/shell.py

@@ -27,7 +27,6 @@ if TYPE_CHECKING:
 class Command(ScrapyCommand):
     default_settings: ClassVar[dict[str, Any]] = {
         "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
-        "KEEP_ALIVE": True,
         "LOGSTATS_INTERVAL": 0,
     }
 

scrapy/core/downloader/__init__.py

@@ -28,6 +28,7 @@ from scrapy.utils.defer import (
     maybe_deferred_to_future,
 )
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.misc import build_from_crawler
 
 if TYPE_CHECKING:
     from collections.abc import Generator
@@ -99,8 +100,8 @@ class Downloader:
         # AUTOTHROTTLE_START_DELAY.
         self._delay: float = self.settings.getfloat("DOWNLOAD_DELAY")
         self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")
-        self.middleware: DownloaderMiddlewareManager = (
+        self.middleware: DownloaderMiddlewareManager = build_from_crawler(
-            DownloaderMiddlewareManager.from_crawler(crawler)
+            DownloaderMiddlewareManager, crawler
         )
         self._slot_gc_loop: AsyncioLoopingCall | LoopingCall | None = None
         self.per_slot_settings: dict[str, dict[str, Any]] = self.settings.getdict(

scrapy/core/downloader/handlers/http11.py

@@ -17,12 +17,19 @@ from twisted.internet.defer import Deferred, succeed
 from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.internet.protocol import Factory, Protocol, connectionDone
 from twisted.python.failure import Failure
+from twisted.web._newclient import (
+    HEADER,
+    STATUS,
+    HTTP11ClientProtocol,
+    HTTPClientParser,
+)
 from twisted.web.client import (
     URI,
     Agent,
     HTTPConnectionPool,
     ResponseDone,
     ResponseFailed,
+    _HTTP11ClientFactory,
 )
 from twisted.web.client import Response as TxResponse
 from twisted.web.http import PotentialDataLoss, _DataLoss
@@ -60,7 +67,8 @@ from ._base_http import BaseHttpDownloadHandler
 
 if TYPE_CHECKING:
     from twisted.internet.base import ReactorBase
-    from twisted.internet.interfaces import IConsumer
+    from twisted.internet.interfaces import IAddress, IConsumer
+    from twisted.web._newclient import Request as TxRequest
 
     # typing.NotRequired requires Python 3.11
     from typing_extensions import NotRequired
@@ -95,7 +103,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
         self._pool.maxPersistentPerHost = crawler.settings.getint(
             "CONCURRENT_REQUESTS_PER_DOMAIN"
         )
-        self._pool._factory.noisy = False
+        self._pool._factory = _LenientHTTP11ClientFactory
 
         self._contextFactory: IPolicyForHTTPS = _load_context_factory_from_settings(
             crawler
@@ -548,7 +556,8 @@ class _ScrapyAgent:
             txresponse._transport._producer.abortConnection()
             raise DownloadCancelledError(warning_msg)
 
-        if warnsize and expected_size > warnsize:
+        reached_warnsize = bool(warnsize and expected_size > warnsize)
+        if reached_warnsize:
             logger.warning(
                 get_warnsize_msg(expected_size, warnsize, request, expected=True)
             )
@@ -561,6 +570,7 @@ class _ScrapyAgent:
                 request=request,
                 maxsize=maxsize,
                 warnsize=warnsize,
+                reached_warnsize=reached_warnsize,
                 fail_on_dataloss=fail_on_dataloss,
                 crawler=self._crawler,
                 tls_verbose_logging=self._tls_verbose_logging,
@@ -625,6 +635,7 @@ class _ResponseReader(Protocol):
         fail_on_dataloss: bool,
         crawler: Crawler,
         *,
+        reached_warnsize: bool = False,
         tls_verbose_logging: bool = False,
     ):
         self._finished: Deferred[_ResultT] = finished
@@ -634,7 +645,7 @@ class _ResponseReader(Protocol):
         self._maxsize: int = maxsize
         self._warnsize: int = warnsize
         self._fail_on_dataloss: bool = fail_on_dataloss
-        self._reached_warnsize: bool = False
+        self._reached_warnsize: bool = reached_warnsize
         self._bytes_received: int = 0
         self._certificate: ssl.Certificate | None = None
         self._ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None = None
@@ -737,3 +748,77 @@ class _ResponseReader(Protocol):
             reason = Failure(exc)
 
         self._finished.errback(reason)
+
+
+class _LenientHTTPClientParser(HTTPClientParser):
+    """Response parser that skips bad response header lines, those with no
+    colon in them, instead of failing to parse the whole response.
+
+    Some servers send such lines, and web browsers skip them and keep parsing
+    the header lines that follow. See
+    https://github.com/scrapy/scrapy/issues/210.
+    """
+
+    def lineReceived(self, line: bytes) -> None:
+        # A copy of twisted.web._newclient.HTTPParser.lineReceived() where the
+        # header name and value are only extracted from header lines that have
+        # a colon.
+
+        # Handle the normal CR LF case.
+        if line[-1:] == b"\r":
+            line = line[:-1]
+
+        if self.state == STATUS:
+            self.statusReceived(line)  # type: ignore[no-untyped-call]
+            self.state = HEADER
+            return
+
+        # HEADER is the only other state in which lines are received, as the
+        # parser switches to raw mode for the response body.
+        if not line or line[0] not in b" \t":
+            if self._partialHeader is not None:
+                header = b"".join(self._partialHeader)
+                if b":" in header:
+                    name, value = header.split(b":", 1)
+                    self.headerReceived(name, value.strip())  # type: ignore[no-untyped-call]
+                else:
+                    logger.debug(
+                        f"Skipping the bad response header line {header!r}, as "
+                        f"it has no colon."
+                    )
+            if not line:
+                # Empty line means the header section is over.
+                self.allHeadersReceived()  # type: ignore[no-untyped-call]
+            else:
+                # Line not beginning with LWS is another header.
+                self._partialHeader = [line]
+        else:
+            # A line beginning with LWS is a continuation of a header begun on
+            # a previous line.
+            self._partialHeader.append(line)  # type: ignore[union-attr]
+
+
+class _LenientHTTP11ClientProtocol(HTTP11ClientProtocol):
+    """Protocol that parses responses with :class:`_LenientHTTPClientParser`."""
+
+    def request(self, request: TxRequest) -> Deferred[IResponse]:
+        d: Deferred[IResponse] = super().request(request)
+        # HTTP11ClientProtocol.request() hardcodes the parser class, so the
+        # only way to use a different one is to replace the class of the parser
+        # object that it creates. This is safe because
+        # _LenientHTTPClientParser defines no additional state. The parser is
+        # always there because HTTPConnectionPool only reuses connections whose
+        # protocol is in the QUIESCENT state, for which request() always
+        # creates a parser.
+        assert self._parser is not None
+        self._parser.__class__ = _LenientHTTPClientParser
+        return d
+
+
+class _LenientHTTP11ClientFactory(_HTTP11ClientFactory):
+    """Factory that builds :class:`_LenientHTTP11ClientProtocol` protocols."""
+
+    noisy = False
+
+    def buildProtocol(self, addr: IAddress | None) -> HTTP11ClientProtocol:
+        return _LenientHTTP11ClientProtocol(self._quiescentCallback)  # type: ignore[no-untyped-call]

scrapy/core/downloader/handlers/http2.py

@@ -40,7 +40,7 @@ class H2DownloadHandler(BaseDownloadHandler):
 
         from twisted.internet import reactor
 
-        self._pool = H2ConnectionPool(reactor, crawler.settings)
+        self._pool = H2ConnectionPool(reactor, crawler)
         self._context_factory = _load_context_factory_from_settings(crawler)
         self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
 

scrapy/core/engine.py

@@ -112,7 +112,6 @@ class ExecutionEngine:
         self.crawler: Crawler = crawler
         self.settings: Settings = crawler.settings
         self.signals: SignalManager = crawler.signals
-        assert crawler.logformatter
         self.logformatter: LogFormatter = crawler.logformatter
         self._slot: _Slot | None = None
         self.spider: Spider | None = None
@@ -125,6 +124,9 @@ class ExecutionEngine:
         ] = spider_closed_callback
         self.start_time: float | None = None
         self._start: AsyncIterator[Any] | None = None
+        # Whether Spider.start() raised, i.e. some start items or requests may
+        # never have reached the engine.
+        self._start_error: bool = False
         self._closewait: Deferred[None] | None = None
         self._start_request_processing_awaitable: (
             asyncio.Future[None] | Deferred[None] | None
@@ -246,7 +248,7 @@ class ExecutionEngine:
         )
         return deferred_from_coro(self.close_async())
 
-    async def close_async(self) -> None:
+    async def close_async(self, *, reason: str = "shutdown") -> None:
         """
         Gracefully close the execution engine.
         If it has already been started, stop it. In all cases, close the spider and the downloader.
@@ -254,9 +256,7 @@ class ExecutionEngine:
         if self.running:
             await self.stop_async()  # will also close spider and downloader
         elif self.spider is not None:
-            await self.close_spider_async(
+            await self.close_spider_async(reason=reason)  # will also close downloader
-                reason="shutdown"
-            )  # will also close downloader
         elif hasattr(self, "downloader"):
             self.downloader.close()
 
@@ -277,13 +277,29 @@ class ExecutionEngine:
             item_or_request = await anext(self._start)
         except StopAsyncIteration:
             self._start = None
+        except CloseSpider as exception:
+            self._start = None
+            _schedule_coro(
+                self.close_spider_async(reason=exception.reason or "cancelled")
+            )
         except Exception as exception:
             self._start = None
+            self._start_error = True
             exception_traceback = format_exc()
             logger.error(
                 f"Error while reading start items and requests: {exception}.\n{exception_traceback}",
                 exc_info=True,
             )
+            self.signals.send_catch_log(
+                signal=signals.spider_error,
+                failure=Failure(),
+                response=None,
+                spider=self.spider,
+            )
+            self.crawler.stats.inc_value("spider_exceptions/count")
+            self.crawler.stats.inc_value(
+                f"spider_exceptions/{type(exception).__name__}"
+            )
         else:
             if not self.spider:
                 return  # spider already closed
@@ -539,24 +555,39 @@ class ExecutionEngine:
         nextcall = CallLaterOnce(self._start_scheduled_requests)
         scheduler = build_from_crawler(self.scheduler_cls, self.crawler)
         self._slot = _Slot(close_if_idle, nextcall, scheduler)
-        self._start = await self.scraper.spidermw.process_start()
+        # A component that fails to start can ask for the spider to be closed.
-        if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)):
+        # The rest of the startup runs anyway, so that components that are
-            await maybe_deferred_to_future(d)
+        # started also get stopped, and the request is honored once the spider
-        await self.scraper.open_spider_async()
+        # is open.
-        assert self.crawler.stats
+        close_spider_exc: CloseSpider | None = None
-        if argument_is_required(self.crawler.stats.open_spider, "spider"):
+        try:
+            self._start = await self.scraper.spidermw.process_start()
+            if hasattr(scheduler, "open") and (
+                d := scheduler.open(self.crawler.spider)
+            ):
+                await maybe_deferred_to_future(d)
+            await self.scraper.open_spider_async()
+        except CloseSpider as exc:
+            close_spider_exc = exc
+        stats = self.crawler.stats
+        if argument_is_required(stats.open_spider, "spider"):
             warnings.warn(
-                f"The open_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"
+                f"The open_spider() method of {global_object_name(type(stats))} requires a spider argument,"
                 f" this is deprecated and the argument will not be passed in future Scrapy versions.",
                 ScrapyDeprecationWarning,
                 stacklevel=2,
             )
-            self.crawler.stats.open_spider(spider=self.crawler.spider)
+            stats.open_spider(spider=self.crawler.spider)
         else:
-            self.crawler.stats.open_spider()
+            stats.open_spider()
-        await self.signals.send_catch_log_async(
+        results = await self.signals.send_catch_log_async(
-            signals.spider_opened, spider=self.crawler.spider
+            signals.spider_opened, spider=self.crawler.spider, dont_log=CloseSpider
         )
+        for _, result in results:
+            if isinstance(result, CloseSpider):
+                close_spider_exc = close_spider_exc or result
+        if close_spider_exc is not None:
+            raise close_spider_exc
 
     def _spider_idle(self) -> None:
         """
@@ -579,7 +610,8 @@ class ExecutionEngine:
         if DontCloseSpider in detected_ex:
             return
         if self.spider_is_idle():
-            ex = detected_ex.get(CloseSpider, CloseSpider(reason="finished"))
+            default_reason = "start_error" if self._start_error else "finished"
+            ex = detected_ex.get(CloseSpider, CloseSpider(reason=default_reason))
             assert isinstance(ex, CloseSpider)  # typing
             _schedule_coro(self.close_spider_async(reason=ex.reason))
 
@@ -655,20 +687,18 @@ class ExecutionEngine:
                 extra={"spider": spider},
             )
 
-        assert self.crawler.stats
         try:
-            if argument_is_required(self.crawler.stats.close_spider, "spider"):
+            stats = self.crawler.stats
+            if argument_is_required(stats.close_spider, "spider"):
                 warnings.warn(
-                    f"The close_spider() method of {global_object_name(type(self.crawler.stats))} requires a spider argument,"
+                    f"The close_spider() method of {global_object_name(type(stats))} requires a spider argument,"
                     f" this is deprecated and the argument will not be passed in future Scrapy versions.",
                     ScrapyDeprecationWarning,
                     stacklevel=2,
                 )
-                self.crawler.stats.close_spider(
+                stats.close_spider(spider=self.crawler.spider, reason=reason)
-                    spider=self.crawler.spider, reason=reason
-                )
             else:
-                self.crawler.stats.close_spider(reason=reason)
+                stats.close_spider(reason=reason)
         except Exception:
             logger.error("Stats close failure")
 

scrapy/core/http2/agent.py

@@ -21,8 +21,8 @@ if TYPE_CHECKING:
     from twisted.internet.base import ReactorBase
     from twisted.internet.endpoints import HostnameEndpoint
 
+    from scrapy.crawler import Crawler
     from scrapy.http import Request, Response
-    from scrapy.settings import Settings
     from scrapy.spiders import Spider
 
 
@@ -30,9 +30,9 @@ ConnectionKeyT = tuple[bytes, bytes, int]
 
 
 class H2ConnectionPool:
-    def __init__(self, reactor: ReactorBase, settings: Settings) -> None:
+    def __init__(self, reactor: ReactorBase, crawler: Crawler) -> None:
         self._reactor = reactor
-        self.settings = settings
+        self._crawler = crawler
 
         # Store a dictionary which is used to get the respective
         # H2ClientProtocolInstance using the  key as Tuple(scheme, hostname, port)
@@ -43,7 +43,7 @@ class H2ConnectionPool:
             ConnectionKeyT, deque[Deferred[H2ClientProtocol]]
         ] = {}
 
-        self._tls_verbose_logging: bool = settings.getbool(
+        self._tls_verbose_logging: bool = crawler.settings.getbool(
             "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
         )
 
@@ -77,7 +77,7 @@ class H2ConnectionPool:
 
         factory = H2ClientFactory(
             uri,
-            self.settings,
+            self._crawler,
             conn_lost_deferred,
             tls_verbose_logging=self._tls_verbose_logging,
         )

scrapy/core/http2/protocol.py

@@ -44,7 +44,7 @@ if TYPE_CHECKING:
     from twisted.python.failure import Failure
     from twisted.web.client import URI
 
-    from scrapy.settings import Settings
+    from scrapy.crawler import Crawler
     from scrapy.spiders import Spider
 
 
@@ -90,7 +90,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
     def __init__(
         self,
         uri: URI,
-        settings: Settings,
+        crawler: Crawler,
         conn_lost_deferred: Deferred[list[BaseException]],
         *,
         tls_verbose_logging: bool = False,
@@ -100,11 +100,12 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             uri -- URI of the base url to which HTTP/2 Connection will be made.
                 uri is used to verify that incoming client requests have correct
                 base URL.
-            settings -- Scrapy project settings
+            crawler -- The crawler the requests belong to
             conn_lost_deferred -- Deferred that fires with the list of underlying exceptions to notify
                 that connection was lost
             tls_verbose_logging -- Whether to log TLS details
         """
+        self._crawler: Crawler = crawler
         self._conn_lost_deferred: Deferred[list[BaseException]] = conn_lost_deferred
         self._tls_verbose_logging: bool = tls_verbose_logging
 
@@ -140,8 +141,8 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             # Both ip_address and uri are used by the Stream before
             # initiating the request to verify that the base address
             # Variables taken from Project Settings
-            "default_download_maxsize": settings.getint("DOWNLOAD_MAXSIZE"),
+            "default_download_maxsize": crawler.settings.getint("DOWNLOAD_MAXSIZE"),
-            "default_download_warnsize": settings.getint("DOWNLOAD_WARNSIZE"),
+            "default_download_warnsize": crawler.settings.getint("DOWNLOAD_WARNSIZE"),
             # Counter to keep track of opened streams. This counter
             # is used to make sure that not more than MAX_CONCURRENT_STREAMS
             # streams are opened which leads to ProtocolError
@@ -208,6 +209,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             stream_id=next(self._stream_id_generator),
             request=request,
             protocol=self,
+            crawler=self._crawler,
             download_maxsize=getattr(
                 spider, "download_maxsize", self.metadata["default_download_maxsize"]
             ),
@@ -461,20 +463,20 @@ class H2ClientFactory(Factory):
     def __init__(
         self,
         uri: URI,
-        settings: Settings,
+        crawler: Crawler,
         conn_lost_deferred: Deferred[list[BaseException]],
         *,
         tls_verbose_logging: bool = False,
     ) -> None:
         self.uri = uri
-        self.settings = settings
+        self.crawler = crawler
         self.conn_lost_deferred = conn_lost_deferred
         self.tls_verbose_logging = tls_verbose_logging
 
     def buildProtocol(self, addr: IAddress) -> H2ClientProtocol:
         return H2ClientProtocol(
             self.uri,
-            self.settings,
+            self.crawler,
             self.conn_lost_deferred,
             tls_verbose_logging=self.tls_verbose_logging,
         )

scrapy/core/http2/stream.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import logging
+from contextlib import suppress
 from enum import Enum
 from io import BytesIO
 from typing import TYPE_CHECKING, Any
@@ -12,9 +13,11 @@ from twisted.internet.error import ConnectionClosed
 from twisted.python.failure import Failure
 from twisted.web.client import ResponseFailed
 
-from scrapy.exceptions import DownloadCancelledError
+from scrapy import signals
+from scrapy.exceptions import DownloadCancelledError, StopDownload
 from scrapy.http.headers import Headers
 from scrapy.utils._download_handlers import (
+    check_stop_download,
     get_maxsize_msg,
     get_warnsize_msg,
     make_response,
@@ -25,6 +28,7 @@ if TYPE_CHECKING:
     from collections.abc import Sequence
 
     from scrapy.core.http2.protocol import H2ClientProtocol
+    from scrapy.crawler import Crawler
     from scrapy.http import Request, Response
 
 
@@ -82,6 +86,9 @@ class StreamCloseReason(Enum):
     # Actual response body size is more than allowed limit
     MAXSIZE_EXCEEDED_ACTUAL = 8
 
+    # A signal handler raised StopDownload
+    STOP_DOWNLOAD = 9
+
 
 class Stream:
     """Represents a single HTTP/2 Stream.
@@ -99,6 +106,7 @@ class Stream:
         stream_id: int,
         request: Request,
         protocol: H2ClientProtocol,
+        crawler: Crawler,
         download_maxsize: int = 0,
         download_warnsize: int = 0,
     ) -> None:
@@ -107,10 +115,13 @@ class Stream:
             stream_id -- Unique identifier for the stream within a single HTTP/2 connection
             request -- The HTTP request associated to the stream
             protocol -- Parent H2ClientProtocol instance
+            crawler -- The crawler the request belongs to
         """
         self.stream_id: int = stream_id
         self._request: Request = request
         self._protocol: H2ClientProtocol = protocol
+        self._crawler: Crawler = crawler
+        self._stop_download: StopDownload | None = None
 
         self._download_maxsize = self._request.meta.get(
             "download_maxsize", download_maxsize
@@ -338,6 +349,13 @@ class Stream:
         self._response["body"].write(data)
         self._response["flow_controlled_size"] += flow_controlled_length
 
+        if stop_download := check_stop_download(
+            signals.bytes_received, self._crawler, self._request, data=data
+        ):
+            self._stop_download = stop_download
+            self.reset_stream(StreamCloseReason.STOP_DOWNLOAD)
+            return
+
         # We check maxsize here in case the Content-Length header was not received
         if (
             self._download_maxsize
@@ -369,8 +387,20 @@ class Stream:
             else:
                 self._response["headers"].appendlist(name, value)
 
-        # Check if we exceed the allowed max data size which can be received
         expected_size = int(self._response["headers"].get(b"Content-Length", -1))
+
+        if stop_download := check_stop_download(
+            signals.headers_received,
+            self._crawler,
+            self._request,
+            headers=self._response["headers"],
+            body_length=expected_size if expected_size >= 0 else None,
+        ):
+            self._stop_download = stop_download
+            self.reset_stream(StreamCloseReason.STOP_DOWNLOAD)
+            return
+
+        # Check if we exceed the allowed max data size which can be received
         if self._download_maxsize and expected_size > self._download_maxsize:
             self.reset_stream(StreamCloseReason.MAXSIZE_EXCEEDED)
             return
@@ -387,11 +417,18 @@ class Stream:
         if self.metadata["stream_closed_local"]:
             raise StreamClosedError(self.stream_id)
 
-        # Clear buffer earlier to avoid keeping data in memory for a long time
+        # The data received so far is the body of the response built for a
-        self._response["body"].truncate(0)
+        # stopped download, otherwise the buffer is cleared early to avoid
+        # keeping data in memory for a long time
+        if reason is not StreamCloseReason.STOP_DOWNLOAD:
+            self._response["body"].truncate(0)
 
         self.metadata["stream_closed_local"] = True
-        self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)
+        # The remote peer may have ended the stream already, e.g. because the
+        # whole response arrived within the data that triggered this reset, in
+        # which case there is nothing left to reset
+        with suppress(StreamClosedError):
+            self._protocol.conn.reset_stream(self.stream_id, ErrorCodes.REFUSED_STREAM)
         self.close(reason)
 
     def close(
@@ -444,7 +481,7 @@ class Stream:
             logger.error(error_msg)
             self._deferred_response.errback(DownloadCancelledError(error_msg))
 
-        elif reason is StreamCloseReason.ENDED:
+        elif reason in {StreamCloseReason.ENDED, StreamCloseReason.STOP_DOWNLOAD}:
             self._fire_response_deferred()
 
         # Stream was abruptly ended here
@@ -495,13 +532,18 @@ class Stream:
         and fires the response deferred callback with the
         generated response instance"""
 
-        response = make_response(
+        try:
-            url=self._request.url,
+            response = make_response(
-            status=self._response["status"],
+                url=self._request.url,
-            headers=self._response["headers"],
+                status=self._response["status"],
-            body=self._response["body"].getvalue(),
+                headers=self._response["headers"],
-            certificate=self._protocol.metadata["certificate"],
+                body=self._response["body"].getvalue(),
-            ip_address=self._protocol.metadata["ip_address"],
+                certificate=self._protocol.metadata["certificate"],
-            protocol="h2",
+                ip_address=self._protocol.metadata["ip_address"],
-        )
+                protocol="h2",
-        self._deferred_response.callback(response)
+                stop_download=self._stop_download,
+            )
+        except StopDownload as exc:
+            self._deferred_response.errback(exc)
+        else:
+            self._deferred_response.callback(response)

scrapy/core/scraper.py

@@ -36,7 +36,11 @@ from scrapy.utils.defer import (
 )
 from scrapy.utils.deprecate import method_is_overridden
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
-from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
+from scrapy.utils.misc import (
+    build_from_crawler,
+    load_object,
+    warn_on_generator_with_return_value,
+)
 from scrapy.utils.python import global_object_name
 from scrapy.utils.spider import iterate_spider_output
 
@@ -102,13 +106,13 @@ class Slot:
 class Scraper:
     def __init__(self, crawler: Crawler) -> None:
         self.slot: Slot | None = None
-        self.spidermw: SpiderMiddlewareManager = SpiderMiddlewareManager.from_crawler(
+        self.spidermw: SpiderMiddlewareManager = build_from_crawler(
-            crawler
+            SpiderMiddlewareManager, crawler
         )
         itemproc_cls: type[ItemPipelineManager] = load_object(
             crawler.settings["ITEM_PROCESSOR"]
         )
-        self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler)
+        self.itemproc: ItemPipelineManager = build_from_crawler(itemproc_cls, crawler)
         self._itemproc_has_async: dict[str, bool] = {}
         for method in [
             "open_spider",
@@ -120,7 +124,6 @@ class Scraper:
         self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS")
         self.crawler: Crawler = crawler
         self.signals: SignalManager = crawler.signals
-        assert crawler.logformatter
         self.logformatter: LogFormatter = crawler.logformatter
 
     def _check_deprecated_itemproc_method(self, method: str) -> None:
@@ -355,7 +358,6 @@ class Scraper:
         assert self.crawler.spider
         exc = _failure.value
         if isinstance(exc, CloseSpider):
-            assert self.crawler.engine is not None  # typing
             _schedule_coro(
                 self.crawler.engine.close_spider_async(reason=exc.reason or "cancelled")
             )
@@ -374,11 +376,9 @@ class Scraper:
             response=response,
             spider=self.crawler.spider,
         )
-        assert self.crawler.stats
+        stats = self.crawler.stats
-        self.crawler.stats.inc_value("spider_exceptions/count")
+        stats.inc_value("spider_exceptions/count")
-        self.crawler.stats.inc_value(
+        stats.inc_value(f"spider_exceptions/{_failure.value.__class__.__name__}")
-            f"spider_exceptions/{_failure.value.__class__.__name__}"
-        )
 
     def handle_spider_output(
         self,
@@ -456,7 +456,6 @@ class Scraper:
         Items are sent to the item pipelines, requests are scheduled.
         """
         if isinstance(output, Request):
-            assert self.crawler.engine is not None  # typing
             self.crawler.engine.crawl(request=output)
             return
         if output is not None:

scrapy/crawler.py

@@ -8,14 +8,14 @@ import signal
 import warnings
 from abc import ABC, abstractmethod
 from functools import partial
-from typing import TYPE_CHECKING, Any, TypeVar
+from typing import TYPE_CHECKING, Any, Generic, TypeVar, overload
 
 from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
 
 from scrapy import Spider
 from scrapy.addons import AddonManager
 from scrapy.core.engine import ExecutionEngine
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning
 from scrapy.extension import ExtensionManager
 from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings
 from scrapy.signalmanager import SignalManager
@@ -58,7 +58,55 @@ logger = logging.getLogger(__name__)
 _T = TypeVar("_T")
 
 
+class _LateAttribute(Generic[_T]):
+    """Descriptor for a :class:`Crawler` attribute that only gets a value once
+    the crawl starts.
+
+    The value is kept in an attribute of the same name prefixed with an
+    underscore, and reading it before it is set raises :exc:`RuntimeError`.
+    This way the public attribute can be annotated as always set, and its
+    users, both in Scrapy and in third-party code, do not need to narrow its
+    type on every use. Code that runs before the crawl starts reads the
+    underscore-prefixed attribute instead.
+    """
+
+    def __set_name__(self, owner: type[Crawler], name: str) -> None:
+        self._name = name
+        self._private_name = f"_{name}"
+
+    @overload
+    def __get__(self, instance: None, owner: type[Crawler]) -> _LateAttribute[_T]: ...
+
+    @overload
+    def __get__(self, instance: Crawler, owner: type[Crawler]) -> _T: ...
+
+    def __get__(
+        self, instance: Crawler | None, owner: type[Crawler]
+    ) -> _LateAttribute[_T] | _T:
+        if instance is None:
+            return self
+        value: _T | None = getattr(instance, self._private_name)
+        if value is None:
+            raise RuntimeError(
+                f"Crawler.{self._name} is not set yet. It is set when the "
+                "crawl starts, so it can only be used from then on, e.g. "
+                "from the spider_opened signal handler onwards."
+            )
+        return value
+
+    def __set__(self, instance: Crawler, value: _T) -> None:
+        setattr(instance, self._private_name, value)
+
+
 class Crawler:
+    engine: _LateAttribute[ExecutionEngine] = _LateAttribute()
+    extensions: _LateAttribute[ExtensionManager] = _LateAttribute()
+    logformatter: _LateAttribute[LogFormatter] = _LateAttribute()
+    request_fingerprinter: _LateAttribute[RequestFingerprinterProtocol] = (
+        _LateAttribute()
+    )
+    stats: _LateAttribute[StatsCollector] = _LateAttribute()
+
     def __init__(
         self,
         spidercls: type[Spider],
@@ -83,12 +131,13 @@ class Crawler:
         self.crawling: bool = False
         self._started: bool = False
 
-        self.extensions: ExtensionManager | None = None
-        self.stats: StatsCollector | None = None
-        self.logformatter: LogFormatter | None = None
-        self.request_fingerprinter: RequestFingerprinterProtocol | None = None
         self.spider: Spider | None = None
-        self.engine: ExecutionEngine | None = None
+
+        self._engine: ExecutionEngine | None = None
+        self._extensions: ExtensionManager | None = None
+        self._logformatter: LogFormatter | None = None
+        self._request_fingerprinter: RequestFingerprinterProtocol | None = None
+        self._stats: StatsCollector | None = None
 
     def _update_root_log_handler(self) -> None:
         if get_scrapy_root_handler() is not None:
@@ -107,7 +156,7 @@ class Crawler:
         self.stats = load_object(self.settings["STATS_CLASS"])(self)
 
         lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
-        self.logformatter = lf_cls.from_crawler(self)
+        self.logformatter = build_from_crawler(lf_cls, self)
 
         self.request_fingerprinter = build_from_crawler(
             load_object(self.settings["REQUEST_FINGERPRINTER_CLASS"]),
@@ -151,7 +200,7 @@ class Crawler:
             logger.debug("Not using a Twisted reactor")
             self._apply_reactorless_default_settings()
 
-        self.extensions = ExtensionManager.from_crawler(self)
+        self.extensions = build_from_crawler(ExtensionManager, self)
         self.settings.freeze()
 
         d = dict(overridden_settings(self.settings))
@@ -221,12 +270,16 @@ class Crawler:
             self._apply_settings()
             self._update_root_log_handler()
             self.engine = self._create_engine()
-            yield deferred_from_coro(self.engine.open_spider_async())
+            try:
-            yield deferred_from_coro(self.engine.start_async())
+                yield deferred_from_coro(self.engine.open_spider_async())
+            except CloseSpider as exc:
+                yield deferred_from_coro(self.engine.close_async(reason=exc.reason))
+            else:
+                yield deferred_from_coro(self.engine.start_async())
         except Exception:
             self.crawling = False
-            if self.engine is not None:
+            if self._engine is not None:
-                yield deferred_from_coro(self.engine.close_async())
+                yield deferred_from_coro(self._engine.close_async())
             raise
 
     async def crawl_async(self, *args: Any, **kwargs: Any) -> None:
@@ -251,12 +304,16 @@ class Crawler:
             self._apply_settings()
             self._update_root_log_handler()
             self.engine = self._create_engine()
-            await self.engine.open_spider_async()
+            try:
-            await self.engine.start_async()
+                await self.engine.open_spider_async()
+            except CloseSpider as exc:
+                await self.engine.close_async(reason=exc.reason)
+            else:
+                await self.engine.start_async()
         except Exception:
             self.crawling = False
-            if self.engine is not None:
+            if self._engine is not None:
-                await self.engine.close_async()
+                await self._engine.close_async()
             raise
 
     def _create_spider(self, *args: Any, **kwargs: Any) -> Spider:
@@ -282,7 +339,6 @@ class Crawler:
         """
         if self.crawling:
             self.crawling = False
-            assert self.engine
             if self.engine.running:
                 await self.engine.stop_async()
 
@@ -313,7 +369,7 @@ class Crawler:
         This method can only be called after the crawl engine has been created,
         e.g. at signals :signal:`engine_started` or :signal:`spider_opened`.
         """
-        if not self.engine:
+        if self._engine is None:
             raise RuntimeError(
                 "Crawler.get_downloader_middleware() can only be called after "
                 "the crawl engine has been created."
@@ -331,7 +387,7 @@ class Crawler:
         created, e.g. at signals :signal:`engine_started` or
         :signal:`spider_opened`.
         """
-        if not self.extensions:
+        if self._extensions is None:
             raise RuntimeError(
                 "Crawler.get_extension() can only be called after the "
                 "extension manager has been created."
@@ -348,7 +404,7 @@ class Crawler:
         This method can only be called after the crawl engine has been created,
         e.g. at signals :signal:`engine_started` or :signal:`spider_opened`.
         """
-        if not self.engine:
+        if self._engine is None:
             raise RuntimeError(
                 "Crawler.get_item_pipeline() can only be called after the "
                 "crawl engine has been created."
@@ -365,7 +421,7 @@ class Crawler:
         This method can only be called after the crawl engine has been created,
         e.g. at signals :signal:`engine_started` or :signal:`spider_opened`.
         """
-        if not self.engine:
+        if self._engine is None:
             raise RuntimeError(
                 "Crawler.get_spider_middleware() can only be called after the "
                 "crawl engine has been created."

scrapy/downloadermiddlewares/httpcache.py

@@ -61,12 +61,10 @@ class HttpCacheMiddleware:
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
-        assert crawler.stats
         o = cls(crawler.settings, crawler.stats)
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         o.crawler = crawler
-        assert crawler.request_fingerprinter
         o._fingerprinter = crawler.request_fingerprinter
         return o
 

scrapy/downloadermiddlewares/httpcompression.py

@@ -30,27 +30,7 @@ if TYPE_CHECKING:
 
 logger = getLogger(__name__)
 
-ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate"]
+ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate", b"br"]
-
-try:
-    try:
-        import brotli
-    except ImportError:
-        import brotlicffi as brotli
-except ImportError:
-    pass
-else:
-    try:
-        brotli.Decompressor.can_accept_more_data  # noqa: B018
-    except AttributeError:  # pragma: no cover
-        warnings.warn(
-            "You have brotli installed. But 'br' encoding support now requires "
-            "brotli's or brotlicffi's version >= 1.2.0. Please upgrade "
-            "brotli/brotlicffi to make Scrapy decode 'br' encoded responses.",
-            stacklevel=2,
-        )
-    else:
-        ACCEPTED_ENCODINGS.append(b"br")
 
 if find_spec("zstandard") is not None:
     ACCEPTED_ENCODINGS.append(b"zstd")
@@ -205,8 +185,6 @@ class HttpCompressionMiddleware:
             f"{self.__class__.__name__} cannot decode the response for {response.url} "
             f"from unsupported encoding(s) '{encodings_str}'."
         )
-        if b"br" in encodings:
-            msg += " You need to install brotli or brotlicffi >= 1.2.0 to decode 'br'."
         if b"zstd" in encodings:
             msg += " You need to install zstandard to decode 'zstd'."
         logger.warning(msg)

scrapy/downloadermiddlewares/offsite.py

@@ -21,15 +21,46 @@ logger = logging.getLogger(__name__)
 
 
 class OffsiteMiddleware:
+    """Filter out requests for URLs outside the domains covered by the spider.
+
+    .. versionadded:: 2.11.2
+
+    A request is allowed if its host name is in the
+    :attr:`~scrapy.Spider.allowed_domains` attribute of the spider, or is a
+    subdomain of one of those domains. E.g. ``www.example.org`` also allows
+    ``bob.www.example.org``, but neither ``www2.example.org`` nor
+    ``example.org``. See :meth:`should_follow` to use a different policy.
+
+    If the spider does not define :attr:`~scrapy.Spider.allowed_domains`, or
+    the attribute is empty, every request is allowed.
+
+    Filtered requests are logged as follows::
+
+        DEBUG: Filtered offsite request to 'offsite.example': <GET http://offsite.example/some/page.html>
+
+    Only the first request filtered for a given domain is logged, to keep the
+    log readable.
+
+    .. reqmeta:: allow_offsite
+
+    allow_offsite
+    -------------
+
+    Requests with the ``allow_offsite`` :attr:`~scrapy.Request.meta` key set to
+    ``True``, or with :attr:`~scrapy.Request.dont_filter` set to ``True``, are
+    allowed regardless of their host name.
+    """
+
     crawler: Crawler
+    host_regex: re.Pattern[str]
 
     def __init__(self, stats: StatsCollector):
         self.stats = stats
         self.domains_seen: set[str] = set()
+        self._allowed_domains: list[str] | None = None
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
-        assert crawler.stats
         o = cls(crawler.stats)
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled)
@@ -37,7 +68,13 @@ class OffsiteMiddleware:
         return o
 
     def spider_opened(self, spider: Spider) -> None:
-        self.host_regex: re.Pattern[str] = self.get_host_regex(spider)
+        self._update_host_regex(spider)
+
+    def _update_host_regex(self, spider: Spider) -> None:
+        allowed_domains = list(getattr(spider, "allowed_domains", None) or [])
+        if allowed_domains != self._allowed_domains:
+            self._allowed_domains = allowed_domains
+            self.host_regex = self.get_host_regex(spider)
 
     def request_scheduled(self, request: Request, spider: Spider) -> None:
         self.process_request(request)
@@ -64,13 +101,30 @@ class OffsiteMiddleware:
         raise IgnoreRequest(f"Filtered offsite request to {domain!r}")
 
     def should_follow(self, request: Request, spider: Spider) -> bool:
+        """Return ``True`` if *request* is on site, ``False`` if it must be
+        filtered out.
+
+        Override this method to implement a different offsite policy. For
+        example, to allow the domains in
+        :attr:`~scrapy.Spider.allowed_domains` but none of their subdomains:
+
+        .. code-block:: python
+
+            from scrapy.downloadermiddlewares.offsite import OffsiteMiddleware
+            from scrapy.utils.httpobj import urlparse_cached
+
+
+            class RootOnlyOffsiteMiddleware(OffsiteMiddleware):
+                def should_follow(self, request, spider):
+                    return urlparse_cached(request).hostname in spider.allowed_domains
+        """
+        self._update_host_regex(spider)
         regex = self.host_regex
         # hostname can be None for wrong urls (like javascript links)
         host = urlparse_cached(request).hostname or ""
         return bool(regex.search(host))
 
     def get_host_regex(self, spider: Spider) -> re.Pattern[str]:
-        """Override this method to implement a different offsite policy"""
         allowed_domains = getattr(spider, "allowed_domains", None)
         if not allowed_domains:
             return re.compile("")  # allow all by default

scrapy/downloadermiddlewares/retry.py

@@ -94,7 +94,6 @@ def get_retry_request(
     retry-related job stats
     """
     settings = spider.crawler.settings
-    assert spider.crawler.stats
     stats = spider.crawler.stats
     retry_times = request.meta.get("retry_times", 0) + 1
     if max_retry_times is None:

scrapy/downloadermiddlewares/robotstxt.py

@@ -18,7 +18,7 @@ from scrapy.http.request import NO_CALLBACK
 from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.misc import load_object
+from scrapy.utils.misc import build_from_crawler, load_object
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
     from scrapy import Spider
     from scrapy.crawler import Crawler
     from scrapy.robotstxt import RobotParser
+    from scrapy.statscollectors import StatsCollector
 
 
 logger = logging.getLogger(__name__)
@@ -41,13 +42,14 @@ class RobotsTxtMiddleware:
         self._default_useragent: str = crawler.settings["USER_AGENT"]
         self._robotstxt_useragent: str | None = crawler.settings["ROBOTSTXT_USER_AGENT"]
         self.crawler: Crawler = crawler
+        self._stats: StatsCollector = crawler.stats
         self._parsers: dict[str, RobotParser | Deferred[RobotParser | None] | None] = {}
         self._parserimpl: RobotParser = load_object(
             crawler.settings.get("ROBOTSTXT_PARSER")
         )
 
         # check if parser dependencies are met, this should throw an error otherwise.
-        self._parserimpl.from_crawler(self.crawler, b"")
+        build_from_crawler(self._parserimpl, self.crawler, b"")
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
@@ -78,8 +80,7 @@ class RobotsTxtMiddleware:
                 {"request": request},
                 extra={"spider": self.crawler.spider},
             )
-            assert self.crawler.stats
+            self._stats.inc_value("robotstxt/forbidden")
-            self.crawler.stats.inc_value("robotstxt/forbidden")
             raise IgnoreRequest("Forbidden by robots.txt")
 
     async def robot_parser(self, request: Request) -> RobotParser | None:
@@ -95,8 +96,6 @@ class RobotsTxtMiddleware:
                 meta={"dont_obey_robotstxt": True},
                 callback=NO_CALLBACK,
             )
-            assert self.crawler.engine
-            assert self.crawler.stats
             try:
                 resp = await self.crawler.engine.download_async(robotsreq)
                 await self._parse_robots(resp, netloc, request)
@@ -109,7 +108,7 @@ class RobotsTxtMiddleware:
                         extra={"spider": self.crawler.spider},
                     )
                 self._robots_error(e, netloc)
-            self.crawler.stats.inc_value("robotstxt/request_count")
+            self._stats.inc_value("robotstxt/request_count")
 
         parser = self._parsers[netloc]
         if isinstance(parser, Deferred):
@@ -119,12 +118,9 @@ class RobotsTxtMiddleware:
     async def _parse_robots(
         self, response: Response, netloc: str, request: Request
     ) -> None:
-        assert self.crawler.stats
+        self._stats.inc_value("robotstxt/response_count")
-        self.crawler.stats.inc_value("robotstxt/response_count")
+        self._stats.inc_value(f"robotstxt/response_status_count/{response.status}")
-        self.crawler.stats.inc_value(
+        rp = build_from_crawler(self._parserimpl, self.crawler, response.body)
-            f"robotstxt/response_status_count/{response.status}"
-        )
-        rp = self._parserimpl.from_crawler(self.crawler, response.body)
         await self.crawler.signals.send_catch_log_async(
             signal=signals.robots_parsed,
             robotparser=rp,
@@ -138,8 +134,7 @@ class RobotsTxtMiddleware:
     def _robots_error(self, exc: Exception, netloc: str) -> None:
         if not isinstance(exc, IgnoreRequest):
             key = f"robotstxt/exception_count/{type(exc)}"
-            assert self.crawler.stats
+            self._stats.inc_value(key)
-            self.crawler.stats.inc_value(key)
         rp_dfd = self._parsers[netloc]
         assert isinstance(rp_dfd, Deferred)
         self._parsers[netloc] = None

scrapy/downloadermiddlewares/stats.py

@@ -43,7 +43,6 @@ class DownloaderStats:
     def from_crawler(cls, crawler: Crawler) -> Self:
         if not crawler.settings.getbool("DOWNLOADER_STATS"):
             raise NotConfigured
-        assert crawler.stats
         return cls(crawler.stats)
 
     @_warn_spider_arg

scrapy/dupefilters.py

@@ -95,7 +95,6 @@ class RFPDupeFilter(BaseDupeFilter):
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
-        assert crawler.request_fingerprinter
         debug = crawler.settings.getbool("DUPEFILTER_DEBUG")
         return cls(
             job_dir(crawler.settings),
@@ -134,5 +133,4 @@ class RFPDupeFilter(BaseDupeFilter):
             self.logger.debug(msg, {"request": request}, extra={"spider": spider})
             self.logdupes = False
 
-        assert spider.crawler.stats
         spider.crawler.stats.inc_value("dupefilter/filtered")

scrapy/exceptions.py

@@ -56,8 +56,11 @@ class DontCloseSpider(Exception):
 
 
 class CloseSpider(Exception):
-    """Raised from a :ref:`spider callback <topics-spiders>` to request the
+    """Raised from a :ref:`spider callback <topics-spiders>`, or while the
-    spider to be closed/stopped.
+    spider is starting, to request the spider to be closed/stopped.
+
+    .. versionchanged:: VERSION
+       Added support for raising it while the spider is starting.
 
     *reason* is a string with the reason for closing.
 

scrapy/exporters.py

@@ -85,11 +85,16 @@ class BaseItemExporter(ABC):
         declared = (name for name in adapter.field_names() if name in populated)
         return dict.fromkeys([*declared, *adapter.keys()])
 
-    def _get_serialized_fields(
+    def get_serialized_fields(
         self, item: Any, default_value: Any = None, include_empty: bool | None = None
     ) -> Iterable[tuple[str, Any]]:
-        """Return the fields to export as an iterable of tuples
+        """Return the fields of *item* to export, as an iterable of
-        (name, serialized_value)
+        ``(name, serialized_value)`` tuples, taking :attr:`fields_to_export`
+        into account and applying :meth:`serialize_field` to every value.
+
+        Fields missing from *item* are exported with *default_value*.
+
+        *include_empty* overrides :attr:`export_empty_fields`.
         """
         item = ItemAdapter(item)
 
@@ -136,7 +141,7 @@ class JsonLinesItemExporter(BaseItemExporter):
         self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs)
 
     def export_item(self, item: Any) -> None:
-        itemdict = dict(self._get_serialized_fields(item))
+        itemdict = dict(self.get_serialized_fields(item))
         data = self.encoder.encode(itemdict) + "\n"
         self.file.write(to_bytes(data, self.encoding))
 
@@ -176,7 +181,7 @@ class JsonItemExporter(BaseItemExporter):
         self.file.write(b"]")
 
     def export_item(self, item: Any) -> None:
-        itemdict = dict(self._get_serialized_fields(item))
+        itemdict = dict(self.get_serialized_fields(item))
         data = to_bytes(self.encoder.encode(itemdict), self.encoding)
         self._add_comma_after_first()
         self.file.write(data)
@@ -216,7 +221,7 @@ class XmlItemExporter(BaseItemExporter):
         self._beautify_indent(depth=1)
         self.xg.startElement(self.item_element, AttributesImpl({}))
         self._beautify_newline()
-        for name, value in self._get_serialized_fields(item, default_value=""):
+        for name, value in self.get_serialized_fields(item, default_value=""):
             self._export_xml_field(name, value, depth=2)
         self._beautify_indent(depth=1)
         self.xg.endElement(self.item_element)
@@ -310,7 +315,7 @@ class CsvItemExporter(BaseItemExporter):
                     f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields",
                 )
                 self._data_loss_warned = True
-        fields = self._get_serialized_fields(item, default_value="", include_empty=True)
+        fields = self.get_serialized_fields(item, default_value="", include_empty=True)
         values = list(self._build_row(x for _, x in fields))
         self.csv_writer.writerow(values)
 
@@ -347,7 +352,7 @@ class PickleItemExporter(BaseItemExporter):
         self.protocol: int = protocol
 
     def export_item(self, item: Any) -> None:
-        d = dict(self._get_serialized_fields(item))
+        d = dict(self.get_serialized_fields(item))
         pickle.dump(d, self.file, self.protocol)
 
 
@@ -365,7 +370,7 @@ class MarshalItemExporter(BaseItemExporter):
         self.file: BytesIO = file
 
     def export_item(self, item: Any) -> None:
-        marshal.dump(dict(self._get_serialized_fields(item)), self.file)
+        marshal.dump(dict(self.get_serialized_fields(item)), self.file)
 
 
 class PprintItemExporter(BaseItemExporter):
@@ -374,7 +379,7 @@ class PprintItemExporter(BaseItemExporter):
         self.file: BytesIO = file
 
     def export_item(self, item: Any) -> None:
-        itemdict = dict(self._get_serialized_fields(item))
+        itemdict = dict(self.get_serialized_fields(item))
         self.file.write(to_bytes(pprint.pformat(itemdict) + "\n"))
 
 
@@ -417,5 +422,5 @@ class PythonItemExporter(BaseItemExporter):
             yield key, self._serialize_value(value)
 
     def export_item(self, item: Any) -> dict[str | bytes, Any]:  # type: ignore[override]
-        result: dict[str | bytes, Any] = dict(self._get_serialized_fields(item))
+        result: dict[str | bytes, Any] = dict(self.get_serialized_fields(item))
         return result

scrapy/extensions/closespider.py

@@ -102,7 +102,6 @@ class CloseSpider:
             self._close_spider("closespider_pagecount_no_item")
 
     def spider_opened(self, spider: Spider) -> None:
-        assert self.crawler.engine
         self.task = call_later(
             self.close_on["timeout"], self._close_spider, "closespider_timeout"
         )
@@ -146,5 +145,4 @@ class CloseSpider:
             self._close_spider("closespider_timeout_no_item")
 
     def _close_spider(self, reason: str) -> None:
-        assert self.crawler.engine
         _schedule_coro(self.crawler.engine.close_spider_async(reason=reason))

scrapy/extensions/corestats.py

@@ -26,7 +26,6 @@ class CoreStats:
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
-        assert crawler.stats
         o = cls(crawler.stats)
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)

scrapy/extensions/debug.py

@@ -45,7 +45,6 @@ class StackTraceDump:
         return cls(crawler)
 
     def dump_stacktrace(self, signum: int, frame: FrameType | None) -> None:
-        assert self.crawler.engine
         log_args = {
             "stackdumps": self._thread_stacks(),
             "enginestatus": format_engine_status(self.crawler.engine),

scrapy/extensions/feedexport.py

@@ -149,7 +149,7 @@ class BlockingFeedStorage(ABC):
 
         return NamedTemporaryFile(prefix="feed-", dir=path)
 
-    def store(self, file: IO[bytes]) -> Deferred[None] | None:
+    def store(self, file: IO[bytes]) -> Deferred[None]:
         return deferred_from_coro(run_in_thread(self._store_in_thread, file))
 
     @abstractmethod
@@ -340,7 +340,7 @@ class GCSFeedStorage(BlockingFeedStorage):
             from google.cloud.storage import Client  # noqa: PLC0415
 
             client = Client(project=self.project_id)
-            bucket = client.get_bucket(self.bucket_name)
+            bucket = client.bucket(self.bucket_name)
             blob = bucket.blob(self.blob_name)
             blob.upload_from_file(file, predefined_acl=self.acl)
         finally:
@@ -363,6 +363,7 @@ class FTPFeedStorage(BlockingFeedStorage):
         self.username: str = u.username or ""
         self.password: str = unquote(u.password or "")
         self.path: str = u.path
+        self.tls: bool = u.scheme == "ftps"
         self.use_active_mode: bool = use_active_mode
         self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)
 
@@ -390,6 +391,7 @@ class FTPFeedStorage(BlockingFeedStorage):
             password=self.password,
             use_active_mode=self.use_active_mode,
             overwrite=self.overwrite,
+            tls=self.tls,
         )
 
 
@@ -611,7 +613,6 @@ class FeedExporter:
 
         logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
         slot_type = type(slot.storage).__name__
-        assert self.crawler.stats
         try:
             await ensure_awaitable(slot.storage.store(self._get_file(slot)))
         except Exception:

scrapy/extensions/httpcache.py

@@ -261,7 +261,6 @@ class DbmCacheStorage:
             extra={"spider": spider},
         )
 
-        assert spider.crawler.request_fingerprinter
         self._fingerprinter: RequestFingerprinterProtocol = (
             spider.crawler.request_fingerprinter
         )
@@ -326,7 +325,6 @@ class FilesystemCacheStorage:
             extra={"spider": spider},
         )
 
-        assert spider.crawler.request_fingerprinter
         self._fingerprinter = spider.crawler.request_fingerprinter
 
     def close_spider(self, spider: Spider) -> None:

scrapy/extensions/logstats.py

@@ -37,7 +37,6 @@ class LogStats:
         interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL")
         if not interval:
             raise NotConfigured
-        assert crawler.stats
         o = cls(crawler.stats, interval)
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)

scrapy/extensions/memdebug.py

@@ -29,7 +29,6 @@ class MemoryDebugger:
     def from_crawler(cls, crawler: Crawler) -> Self:
         if not crawler.settings.getbool("MEMDEBUG_ENABLED"):
             raise NotConfigured
-        assert crawler.stats
         o = cls(crawler.stats)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         return o

scrapy/extensions/memusage.py

@@ -19,6 +19,7 @@ from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
 from scrapy.utils.defer import _schedule_coro
 from scrapy.utils.engine import get_engine_status
+from scrapy.utils.misc import build_from_crawler
 
 if TYPE_CHECKING:
     from twisted.internet.task import LoopingCall
@@ -27,6 +28,7 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
     from scrapy.crawler import Crawler
+    from scrapy.statscollectors import StatsCollector
 
 
 logger = logging.getLogger(__name__)
@@ -43,6 +45,7 @@ class MemoryUsage:
             raise NotConfigured from exc
 
         self.crawler: Crawler = crawler
+        self._stats: StatsCollector = crawler.stats
         self.warned: bool = False
         self.notify_mails: list[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")
         if self.notify_mails:  # pragma: no cover
@@ -55,7 +58,7 @@ class MemoryUsage:
                 category=ScrapyDeprecationWarning,
                 stacklevel=2,
             )
-            self.mail = MailSender.from_crawler(crawler)
+            self.mail = build_from_crawler(MailSender, crawler)
 
         self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024
         self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024
@@ -77,8 +80,7 @@ class MemoryUsage:
         return size
 
     def engine_started(self) -> None:
-        assert self.crawler.stats
+        self._stats.set_value("memusage/startup", self.get_virtual_size())
-        self.crawler.stats.set_value("memusage/startup", self.get_virtual_size())
         self.tasks: list[AsyncioLoopingCall | LoopingCall] = []
         tsk = create_looping_call(self.update)
         self.tasks.append(tsk)
@@ -98,15 +100,12 @@ class MemoryUsage:
                 tsk.stop()
 
     def update(self) -> None:
-        assert self.crawler.stats
+        self._stats.max_value("memusage/max", self.get_virtual_size())
-        self.crawler.stats.max_value("memusage/max", self.get_virtual_size())
 
     def _check_limit(self) -> None:
-        assert self.crawler.engine
-        assert self.crawler.stats
         peak_mem_usage = self.get_virtual_size()
         if peak_mem_usage > self.limit:
-            self.crawler.stats.set_value("memusage/limit_reached", 1)
+            self._stats.set_value("memusage/limit_reached", 1)
             mem = self.limit / 1024 / 1024
             logger.error(
                 "Memory usage exceeded %(memusage)dMiB. Shutting down Scrapy...",
@@ -119,7 +118,7 @@ class MemoryUsage:
                     f"memory usage exceeded {mem}MiB at {socket.gethostname()}"
                 )
                 self._send_report(self.notify_mails, subj)
-                self.crawler.stats.set_value("memusage/limit_notified", 1)
+                self._stats.set_value("memusage/limit_notified", 1)
 
             if self.crawler.engine.spider is not None:
                 _schedule_coro(
@@ -136,9 +135,8 @@ class MemoryUsage:
     def _check_warning(self) -> None:
         if self.warned:  # warn only once
             return
-        assert self.crawler.stats
         if self.get_virtual_size() > self.warning:
-            self.crawler.stats.set_value("memusage/warning_reached", 1)
+            self._stats.set_value("memusage/warning_reached", 1)
             self.crawler.signals.send_catch_log(signal=signals.memusage_warning_reached)
             mem = self.warning / 1024 / 1024
             logger.warning(
@@ -152,16 +150,13 @@ class MemoryUsage:
                     f"memory usage reached {mem}MiB at {socket.gethostname()}"
                 )
                 self._send_report(self.notify_mails, subj)
-                self.crawler.stats.set_value("memusage/warning_notified", 1)
+                self._stats.set_value("memusage/warning_notified", 1)
             self.warned = True
 
     def _send_report(self, rcpts: list[str], subject: str) -> None:  # pragma: no cover
         """send notification mail with some additional useful info"""
-        assert self.crawler.engine
+        s = f"Memory usage at engine startup : {self._stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
-        assert self.crawler.stats
+        s += f"Maximum memory usage          : {self._stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
-        stats = self.crawler.stats
-        s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
-        s += f"Maximum memory usage          : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
         s += f"Current memory usage          : {self.get_virtual_size() / 1024 / 1024}M\r\n"
 
         s += (

scrapy/extensions/periodic_log.py

@@ -87,7 +87,6 @@ class PeriodicLog:
         )
         if not (ext_stats or ext_delta or ext_timing_enabled):
             raise NotConfigured
-        assert crawler.stats
         assert ext_stats is not None
         assert ext_delta is not None
         o = cls(

scrapy/extensions/statsmailer.py

@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
 from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.mail import MailSender
+from scrapy.utils.misc import build_from_crawler
 
 if TYPE_CHECKING:
     from twisted.internet.defer import Deferred
@@ -41,8 +42,7 @@ class StatsMailer:
         recipients: list[str] = crawler.settings.getlist("STATSMAILER_RCPTS")
         if not recipients:
             raise NotConfigured
-        mail: MailSender = MailSender.from_crawler(crawler)
+        mail: MailSender = build_from_crawler(MailSender, crawler)
-        assert crawler.stats
         o = cls(crawler.stats, recipients, mail)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         return o

scrapy/extensions/telnet.py

@@ -52,6 +52,7 @@ class TelnetConsole(protocol.ServerFactory):
 
         self.crawler: Crawler = crawler
         self.noisy: bool = False
+        self.port: Port | None = None
         self.portrange: list[int] = [
             int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT")
         ]
@@ -71,7 +72,7 @@ class TelnetConsole(protocol.ServerFactory):
         return cls(crawler)
 
     def start_listening(self) -> None:
-        self.port: Port = listen_tcp(self.portrange, self.host, self)
+        self.port = listen_tcp(self.portrange, self.host, self)
         h = self.port.getHost()
         logger.info(
             "Telnet console listening on %(host)s:%(port)d",
@@ -80,7 +81,10 @@ class TelnetConsole(protocol.ServerFactory):
         )
 
     def stop_listening(self) -> None:
-        self.port.stopListening()
+        # The port is unset if start_listening() failed, e.g. because every
+        # port in TELNETCONSOLE_PORT was taken.
+        if self.port is not None:
+            self.port.stopListening()
 
     def protocol(self) -> telnet.TelnetTransport:
         class Portal:
@@ -104,7 +108,6 @@ class TelnetConsole(protocol.ServerFactory):
 
     def _get_telnet_vars(self) -> dict[str, Any]:
         # Note: if you add entries here also update topics/telnetconsole.rst
-        assert self.crawler.engine
         telnet_vars: dict[str, Any] = {
             "engine": self.crawler.engine,
             "spider": self.crawler.engine.spider,

scrapy/extensions/throttle.py

@@ -45,7 +45,6 @@ class AutoThrottle:
     def _spider_opened(self, spider: Spider) -> None:
         self.mindelay = self._min_delay()
         self.maxdelay = self._max_delay()
-        assert self.crawler.engine
         self.crawler.engine.downloader._delay = self._start_delay()
 
     def _min_delay(self) -> float:
@@ -98,7 +97,6 @@ class AutoThrottle:
         key: str | None = request.meta.get("download_slot")
         if key is None:
             return None, None
-        assert self.crawler.engine
         return key, self.crawler.engine.downloader.slots.get(key)
 
     def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None:

scrapy/http/cookies.py

@@ -54,9 +54,9 @@ class CookieJar:
         if not IPV4_RE.search(req_host):
             hosts = potential_domain_matches(req_host)
             if "." not in req_host:
-                hosts.append(req_host + ".local")
+                hosts += potential_domain_matches(req_host + ".local")
         else:
-            hosts = [req_host]
+            hosts = [req_host, "." + req_host]
 
         cookies = []
         for host in hosts:

scrapy/http/response/__init__.py

@@ -305,6 +305,11 @@ class Response(object_ref):
         :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all`
         method which supports selectors in addition to absolute/relative URLs
         and Link objects.
+
+        .. caution:: Every returned request gets its own *meta* and
+            *cb_kwargs* dictionaries, but the values within them are shared.
+            Mutating one of those values, e.g. appending to a list, affects
+            all the returned requests.
         """
         if not hasattr(urls, "__iter__"):
             raise TypeError("'urls' argument must be an iterable")

scrapy/http/response/text.py

@@ -84,9 +84,21 @@ class TextResponse(Response):
         )
 
     def json(self) -> Any:
-        """Deserialize a JSON document to a Python object."""
+        """Deserialize a JSON document to a Python object.
+
+        .. versionchanged:: VERSION
+           Bodies that cannot be decoded as UTF-8, UTF-16 or UTF-32, as the
+           JSON specification requires, are now decoded using
+           :attr:`TextResponse.encoding` instead of raising
+           :exc:`UnicodeDecodeError`.
+
+        The result is cached after the first call.
+        """
         if self._cached_decoded_json is _NONE:
-            self._cached_decoded_json = json.loads(self.body)
+            try:
+                self._cached_decoded_json = json.loads(self.body)
+            except UnicodeDecodeError:
+                self._cached_decoded_json = json.loads(self.text)
         return self._cached_decoded_json
 
     @property
@@ -264,6 +276,9 @@ class TextResponse(Response):
         using the ``css`` or ``xpath`` parameters, this method will not produce requests for
         selectors from which links cannot be obtained (for instance, anchor tags without an
         ``href`` attribute)
+
+        .. seealso:: :meth:`.Response.follow_all`, for a caution about mutable
+            *meta* and *cb_kwargs* values.
         """
         arguments = [x for x in (urls, css, xpath) if x is not None]
         if len(arguments) != 1:

scrapy/linkextractors/lxmlhtml.py

@@ -282,6 +282,12 @@ class LxmlLinkExtractor:
                 if m:
                     return m.group(1)
 
+        ``process_value`` is called before the filtering parameters, such as
+        ``allow`` and ``deny``, which match the value that it returns. To drop
+        links based on their final URL, use the ``process_links`` parameter of
+        :class:`~scrapy.spiders.Rule`, which only receives links that those
+        parameters kept.
+
     :type process_value: collections.abc.Callable
 
     :param strip: whether to strip whitespaces from extracted attributes.
@@ -326,7 +332,7 @@ class LxmlLinkExtractor:
             unique=unique,
             process=process_value,
             strip=strip,
-            canonicalized=not canonicalize,
+            canonicalized=True,
         )
         self.allow_res: list[re.Pattern[str]] = self._compile_regexes(allow)
         self.deny_res: list[re.Pattern[str]] = self._compile_regexes(deny)

scrapy/pipelines/files.py

@@ -27,9 +27,7 @@ from twisted.internet.defer import Deferred, maybeDeferred
 from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
-from scrapy.pipelines.media import (
+from scrapy.pipelines.media import FileException as _FileException
-    FileException as FileException,  # noqa: PLC0414  # re-exported for backward compatibility
-)
 from scrapy.pipelines.media import (
     FileInfo,
     FileInfoOrError,
@@ -626,7 +624,7 @@ class FilesPipeline(MediaPipeline):
             f"{request} referred in <{referer}>: {failure.value}",
             extra={"spider": info.spider},
         )
-        raise FileException
+        raise _FileException
 
     async def media_downloaded(
         self,
@@ -645,7 +643,7 @@ class FilesPipeline(MediaPipeline):
                 {"status": response.status, "request": request, "referer": referer},
                 extra={"spider": info.spider},
             )
-            raise FileException("download-error")
+            raise _FileException("download-error")
 
         if not response.body:
             logger.warning(
@@ -654,7 +652,7 @@ class FilesPipeline(MediaPipeline):
                 {"request": request, "referer": referer},
                 extra={"spider": info.spider},
             )
-            raise FileException("empty-content")
+            raise _FileException("empty-content")
 
         status = "cached" if "cached" in response.flags else "downloaded"
         logger.debug(
@@ -670,7 +668,7 @@ class FilesPipeline(MediaPipeline):
             checksum: str = await ensure_awaitable(
                 self.file_downloaded(response, request, info, item=item)
             )
-        except FileException as exc:
+        except _FileException as exc:
             logger.warning(
                 "File (error): Error processing file from %(request)s "
                 "referred in <%(referer)s>: %(errormsg)s",
@@ -687,7 +685,7 @@ class FilesPipeline(MediaPipeline):
                 exc_info=True,
                 extra={"spider": info.spider},
             )
-            raise FileException(str(exc)) from exc
+            raise _FileException(str(exc)) from exc
 
         return {
             "url": request.url,
@@ -697,9 +695,9 @@ class FilesPipeline(MediaPipeline):
         }
 
     def inc_stats(self, status: str) -> None:
-        assert self.crawler.stats
+        stats = self.crawler.stats
-        self.crawler.stats.inc_value("file_count")
+        stats.inc_value("file_count")
-        self.crawler.stats.inc_value(f"file_status_count/{status}")
+        stats.inc_value(f"file_status_count/{status}")
 
     async def _file_downloaded(
         self,
@@ -770,3 +768,15 @@ class FilesPipeline(MediaPipeline):
             if media_type:
                 media_ext = cast("str", mimetypes.guess_extension(media_type))
         return f"full/{media_guid}{media_ext}"
+
+
+def __getattr__(name: str) -> Any:
+    if name == "FileException":
+        warnings.warn(
+            "scrapy.pipelines.files.FileException is deprecated, use "
+            "scrapy.pipelines.media.FileException instead.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return _FileException
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

scrapy/pipelines/images.py

@@ -18,18 +18,13 @@ from itemadapter import ItemAdapter
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
-from scrapy.pipelines.files import (
+from scrapy.pipelines.files import FilesPipeline, GCSFilesStore, S3FilesStore, _md5sum
-    FileException,
+from scrapy.pipelines.media import FileException
-    FilesPipeline,
-    GCSFilesStore,
-    S3FilesStore,
-    _md5sum,
-)
 from scrapy.utils.defer import ensure_awaitable
 from scrapy.utils.python import to_bytes
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable
+    from collections.abc import Iterator
     from os import PathLike
 
     from PIL import Image
@@ -180,7 +175,7 @@ class ImagesPipeline(FilesPipeline):
         info: MediaPipeline.SpiderInfo,
         *,
         item: Any = None,
-    ) -> Iterable[tuple[str, Image.Image, BytesIO]]:
+    ) -> Iterator[tuple[str, Image.Image, BytesIO]]:
         path = self.file_path(request, response=response, info=info, item=item)
         orig_image = self._Image.open(BytesIO(response.body))
         transposed_image = self._ImageOps.exif_transpose(orig_image)

scrapy/pipelines/media.py

@@ -100,7 +100,6 @@ class MediaPipeline(ABC):
                 stacklevel=2,
             )
         self.crawler: Crawler = crawler
-        assert crawler.request_fingerprinter
         self._fingerprinter: RequestFingerprinterProtocol = (
             crawler.request_fingerprinter
         )
@@ -228,7 +227,6 @@ class MediaPipeline(ABC):
     ) -> FileInfo:
         try:
             self._modify_media_request(request)
-            assert self.crawler.engine
             response = await self.crawler.engine.download_async(request)
             return await ensure_awaitable(
                 self.media_downloaded(response, request, info, item=item)

scrapy/pqueues.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 import hashlib
 import logging
+from contextlib import suppress
+from pathlib import Path
 from typing import TYPE_CHECKING, Protocol, cast
 
 from scrapy.utils.misc import build_from_crawler
@@ -263,7 +265,6 @@ class ScrapyPriorityQueue:
 
 class DownloaderInterface:
     def __init__(self, crawler: Crawler):
-        assert crawler.engine
         self.downloader: Downloader = crawler.engine.downloader
 
     def stats(self, possible_slots: Iterable[str]) -> list[tuple[int, str]]:
@@ -409,6 +410,11 @@ class DownloaderAwarePriorityQueue:
         request = queue.pop()
         if len(queue) == 0:
             del self.pqueues[slot]
+            if self.key:
+                # Reclaim the slot directory; rmdir leaves it alone if the
+                # downstream queues did not remove all their files.
+                with suppress(OSError):
+                    Path(self.key, _path_safe(slot)).rmdir()
         return request
 
     def push(self, request: Request) -> None:

scrapy/selector/unified.py

@@ -46,8 +46,10 @@ class Selector(_ParselSelector, object_ref):
     ``"json"``, ``"text"`` or ``None`` (default). It's passed to
     :class:`parsel.Selector` and its meaning is defined there. However, when
     ``type`` is ``None``, it is set to ``"xml"`` for an
-    :class:`~scrapy.http.XmlResponse` and to ``"html"`` otherwise before
+    :class:`~scrapy.http.XmlResponse` and to ``"html"`` for an
-    passing it to :class:`parsel.Selector`.
+    :class:`~scrapy.http.HtmlResponse` or for ``text`` before passing it to
+    :class:`parsel.Selector`, which for any other response is left to
+    determine the type from the response body.
 
     .. note:: JSON selector support requires ``parsel`` 1.8.0 or higher. With
        older versions setting ``type`` to ``"json"`` or ``"text"`` is not
@@ -70,8 +72,13 @@ class Selector(_ParselSelector, object_ref):
                 f"{self.__class__.__name__}.__init__() received both response and text"
             )
 
+        # A response that is neither HTML nor XML, e.g. a JSON one, keeps type
+        # unset, so that parsel determines it from the body.
         if type is None:
-            type = "xml" if isinstance(response, XmlResponse) else "html"  # noqa: A001
+            if isinstance(response, XmlResponse):
+                type = "xml"  # noqa: A001
+            elif response is None or isinstance(response, HtmlResponse):
+                type = "html"  # noqa: A001
 
         if text is not None:
             response = _response_from_text(text, type)

scrapy/settings/default_settings.py

@@ -378,6 +378,7 @@ FEED_STORAGES_BASE = {
     "": "scrapy.extensions.feedexport.FileFeedStorage",
     "file": "scrapy.extensions.feedexport.FileFeedStorage",
     "ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
+    "ftps": "scrapy.extensions.feedexport.FTPFeedStorage",
     "gs": "scrapy.extensions.feedexport.GCSFeedStorage",
     "s3": "scrapy.extensions.feedexport.S3FeedStorage",
     "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",

scrapy/shell.py

@@ -193,7 +193,6 @@ class Shell:
         """
         if not self.spider:
             await self._open_spider(spider)
-        assert self.crawler.engine is not None
         # send the request to the engine
         self.crawler.engine.crawl(request)
         # this will fire when the request callback runs (via the callback hijacking in _request_deferred())
@@ -204,7 +203,6 @@ class Shell:
             spider = self.crawler.spider or self.crawler._create_spider()
 
         self.crawler.spider = spider
-        assert self.crawler.engine
         await self.crawler.engine.open_spider_async(close_if_idle=False)
         self.spider = spider
 

scrapy/spidermiddlewares/depth.py

@@ -28,6 +28,27 @@ logger = logging.getLogger(__name__)
 
 
 class DepthMiddleware(BaseSpiderMiddleware):
+    """Track the depth of each request within the site being scraped, setting
+    ``request.meta["depth"]`` to 0 when there is no value previously set
+    (usually just the first request) and incrementing it by 1 otherwise.
+
+    It can be used to limit the maximum depth to scrape, control request
+    priority based on their depth, and things like that, through the
+    :setting:`DEPTH_LIMIT`, :setting:`DEPTH_STATS_VERBOSE` and
+    :setting:`DEPTH_PRIORITY` settings.
+
+    .. reqmeta:: depth_reset
+
+    depth_reset
+    -----------
+
+    .. versionadded:: VERSION
+
+    :attr:`~scrapy.Request.meta` key that, set to ``True``, gives a request
+    depth 0 instead of the depth of its source response plus 1, e.g. to keep
+    :setting:`DEPTH_LIMIT` from applying across a domain change.
+    """
+
     crawler: Crawler
 
     def __init__(  # pylint: disable=super-init-not-called
@@ -41,6 +62,7 @@ class DepthMiddleware(BaseSpiderMiddleware):
         self.stats = stats
         self.verbose_stats = verbose_stats
         self.prio = prio
+        self._ignored_logged = False
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
@@ -48,7 +70,6 @@ class DepthMiddleware(BaseSpiderMiddleware):
         maxdepth = settings.getint("DEPTH_LIMIT")
         verbose = settings.getbool("DEPTH_STATS_VERBOSE")
         prio = settings.getint("DEPTH_PRIORITY")
-        assert crawler.stats
         o = cls(maxdepth, crawler.stats, verbose, prio)
         o.crawler = crawler
         return o
@@ -86,19 +107,25 @@ class DepthMiddleware(BaseSpiderMiddleware):
     def get_processed_request(
         self, request: Request, response: Response | None
     ) -> Request | None:
+        # Consumed here so that it cannot reach response.meta and, from there,
+        # spread to further requests through a meta copy.
+        depth_reset = request.meta.pop("depth_reset", False)
         if response is None:
             # start requests
             return request
-        depth = response.meta["depth"] + 1
+        depth = 0 if depth_reset else response.meta["depth"] + 1
         request.meta["depth"] = depth
         if self.prio:
             request.priority -= depth * self.prio
         if self.maxdepth and depth > self.maxdepth:
-            logger.debug(
+            if not self._ignored_logged:
-                "Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
+                logger.debug(
-                {"maxdepth": self.maxdepth, "requrl": request.url},
+                    f"Ignoring link (depth > {self.maxdepth}): {request.url}"
-                extra={"spider": self.crawler.spider},
+                    " - no more ignored links will be shown",
-            )
+                    extra={"spider": self.crawler.spider},
+                )
+                self._ignored_logged = True
+            self.stats.inc_value("depth/request_ignored_count")
             return None
         if self.verbose_stats:
             self.stats.inc_value(f"request_depth_count/{depth}")

scrapy/spidermiddlewares/httperror.py

@@ -78,9 +78,9 @@ class HttpErrorMiddleware:
         self, response: Response, exception: Exception, spider: Spider | None = None
     ) -> Iterable[Any] | None:
         if isinstance(exception, HttpError):
-            assert self.crawler.stats
+            stats = self.crawler.stats
-            self.crawler.stats.inc_value("httperror/response_ignored_count")
+            stats.inc_value("httperror/response_ignored_count")
-            self.crawler.stats.inc_value(
+            stats.inc_value(
                 f"httperror/response_ignored_status_count/{response.status}"
             )
             logger.info(

scrapy/spidermiddlewares/metacopy.py

@@ -15,8 +15,7 @@ logger = logging.getLogger(__name__)
 
 class MetaCopyDetectionMiddleware(BaseSpiderMiddleware):
     """Warn when a spider yields a request with internal meta keys that should
-    not be copied from response.meta, or when two requests share the same meta
+    not be copied from response.meta.
-    dict object.
 
     Each warning is emitted at most once per crawl.
     """

scrapy/spidermiddlewares/urllength.py

@@ -48,6 +48,5 @@ class UrlLengthMiddleware(BaseSpiderMiddleware):
             {"maxlength": self.maxlength, "url": request.url},
             extra={"spider": self.crawler.spider},
         )
-        assert self.crawler.stats
         self.crawler.stats.inc_value("urllength/request_ignored_count")
         return None

scrapy/utils/_compression.py

@@ -2,11 +2,10 @@ import contextlib
 import zlib
 from io import BytesIO
 
-with contextlib.suppress(ImportError):
+try:
-    try:
+    import brotli
-        import brotli
+except ImportError:
-    except ImportError:
+    import brotlicffi as brotli
-        import brotlicffi as brotli
 
 with contextlib.suppress(ImportError):
     import zstandard

scrapy/utils/ftp.py

@@ -1,7 +1,8 @@
 import posixpath
 from contextlib import closing
-from ftplib import FTP, error_perm
+from ftplib import FTP, FTP_TLS, error_perm
 from posixpath import dirname
+from ssl import create_default_context
 from typing import IO
 
 
@@ -29,13 +30,20 @@ def ftp_store_file(
     password: str,
     use_active_mode: bool = False,
     overwrite: bool = True,
+    tls: bool = False,
 ) -> None:
-    """Opens a FTP connection with passed credentials,sets current directory
+    """Opens a FTP connection with passed credentials, sets current directory
-    to the directory extracted from given path, then uploads the file to server
+    to the directory extracted from given path, then uploads the file to server.
+
+    If *tls* is ``True``, the connection is secured with TLS (FTPS), and the
+    certificate of the server is verified.
     """
-    with FTP() as ftp, closing(file):
+    ftp = FTP_TLS(context=create_default_context()) if tls else FTP()
+    with ftp, closing(file):
         ftp.connect(host, port)
         ftp.login(username, password)
+        if isinstance(ftp, FTP_TLS):
+            ftp.prot_p()
         if use_active_mode:
             ftp.set_pasv(False)
         file.seek(0)

scrapy/utils/log.py

@@ -2,7 +2,9 @@ from __future__ import annotations
 
 import logging
 import pprint
+import re
 import sys
+import warnings
 from collections.abc import MutableMapping
 from logging.config import dictConfig
 from typing import TYPE_CHECKING, Any, cast
@@ -12,6 +14,7 @@ from twisted.python import log as twisted_log
 from twisted.python.failure import Failure
 
 import scrapy
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings
 from scrapy.utils.versions import get_versions
 
@@ -239,13 +242,15 @@ class LogCounterHandler(logging.Handler):
 
     def emit(self, record: logging.LogRecord) -> None:
         sname = f"log_count/{record.levelname}"
-        assert self.crawler.stats
         self.crawler.stats.inc_value(sname)
 
 
+_MSG_MAPPING_PLACEHOLDER = re.compile(r"%\(\w+\)")
+
+
 def logformatter_adapter(
     logkws: LogFormatterResult,
-) -> tuple[int, str, dict[str, Any] | tuple[Any, ...]]:
+) -> tuple[Any, ...]:
     """
     Helper that takes the dictionary output from the methods in LogFormatter
     and adapts it into a tuple of positional arguments for logger.log calls.
@@ -253,10 +258,28 @@ def logformatter_adapter(
 
     level = logkws.get("level", logging.INFO)
     message = logkws.get("msg") or ""
-    # NOTE: This also handles 'args' being an empty dict, that case doesn't
+    args = logkws.get("args")
-    # play well in logger.log calls
+    # logging interpolates the message whenever it receives any positional
-    args = cast("dict[str, Any]", logkws) if not logkws.get("args") else logkws["args"]
+    # argument, so empty args are left out. Tuple args become one positional
-
+    # argument each, while a dict is a single positional argument.
+    if not args:
+        if _MSG_MAPPING_PLACEHOLDER.search(message):
+            # The log formatter method has already returned, so there is no
+            # frame of it left in the stack to point at. msg is part of the
+            # warning message instead, so that each offending method gets its
+            # own warning.
+            warnings.warn(
+                f"A log formatter method returned msg {message!r} with "
+                f"%(name)s placeholders and no args. Interpolating msg with "
+                f"the returned dict is deprecated, return those values under "
+                f"args instead.",
+                ScrapyDeprecationWarning,
+                stacklevel=1,
+            )
+            return (level, message, logkws)
+        return (level, message)
+    if isinstance(args, tuple):
+        return (level, message, *args)
     return (level, message, args)
 
 

scrapy/utils/response.py

@@ -90,6 +90,10 @@ def open_in_browser(
         def parse_details(self, response):
             if "item name" not in response.text:
                 open_in_browser(response)
+
+    On the Windows Subsystem for Linux, set the ``BROWSER`` environment
+    variable to `wslview <https://github.com/wslutilities/wslu>`_ to open the
+    response in a Windows browser, which cannot read Linux paths otherwise.
     """
     # circular imports
     from scrapy.http import HtmlResponse, TextResponse  # noqa: PLC0415

scrapy/utils/serialize.py

@@ -10,18 +10,11 @@ from scrapy.http import Request, Response
 
 
 class ScrapyJSONEncoder(json.JSONEncoder):
-    DATE_FORMAT = "%Y-%m-%d"
-    TIME_FORMAT = "%H:%M:%S"
-
     def default(self, o: Any) -> Any:
         if isinstance(o, set):
             return list(o)
-        if isinstance(o, datetime.datetime):
+        if isinstance(o, (datetime.datetime, datetime.date, datetime.time)):
-            return o.strftime(f"{self.DATE_FORMAT} {self.TIME_FORMAT}")
+            return o.isoformat()
-        if isinstance(o, datetime.date):
-            return o.strftime(self.DATE_FORMAT)
-        if isinstance(o, datetime.time):
-            return o.strftime(self.TIME_FORMAT)
         if isinstance(o, decimal.Decimal):
             return str(o)
         if isinstance(o, defer.Deferred):

scrapy/utils/trackref.py

@@ -14,7 +14,6 @@ This library has a minimal performance impact.
 
 from __future__ import annotations
 
-from collections import defaultdict
 from operator import itemgetter
 from time import monotonic_ns
 from types import NoneType
@@ -28,8 +27,8 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
 
-live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict(
+live_refs: WeakKeyDictionary[type, WeakKeyDictionary[object, float]] = (
-    WeakKeyDictionary
+    WeakKeyDictionary()
 )
 
 
@@ -41,7 +40,11 @@ class object_ref:
 
     def __new__(cls, *args: Any, **kwargs: Any) -> Self:
         obj = object.__new__(cls)
-        live_refs[cls][obj] = monotonic_ns()
+        try:
+            refs = live_refs[cls]
+        except KeyError:
+            refs = live_refs[cls] = WeakKeyDictionary()
+        refs[obj] = monotonic_ns()
         return obj
 
 

tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py

@@ -8,7 +8,10 @@ class CachingHostnameResolverSpider(scrapy.Spider):
     """
 
     name = "caching_hostname_resolver_spider"
-    start_urls = ["http://[::1]"]
+
+    async def start(self):
+        # w3lib older than 2.4.1 strips the brackets, making the URL invalid.
+        yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})
 
 
 if __name__ == "__main__":

tests/AsyncCrawlerProcess/default_name_resolver.py

@@ -9,7 +9,10 @@ class IPv6Spider(scrapy.Spider):
     """
 
     name = "ipv6_spider"
-    start_urls = ["http://[::1]"]
+
+    async def start(self):
+        # w3lib older than 2.4.1 strips the brackets, making the URL invalid.
+        yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})
 
 
 if __name__ == "__main__":

tests/CrawlerProcess/caching_hostname_resolver_ipv6.py

@@ -8,7 +8,10 @@ class CachingHostnameResolverSpider(scrapy.Spider):
     """
 
     name = "caching_hostname_resolver_spider"
-    start_urls = ["http://[::1]"]
+
+    async def start(self):
+        # w3lib older than 2.4.1 strips the brackets, making the URL invalid.
+        yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})
 
 
 if __name__ == "__main__":

tests/CrawlerProcess/default_name_resolver.py

@@ -9,7 +9,10 @@ class IPv6Spider(scrapy.Spider):
     """
 
     name = "ipv6_spider"
-    start_urls = ["http://[::1]"]
+
+    async def start(self):
+        # w3lib older than 2.4.1 strips the brackets, making the URL invalid.
+        yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})
 
 
 if __name__ == "__main__":

tests/benchmarks/test_crawl.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import asyncio
+from collections import Counter
 from typing import TYPE_CHECKING, Any
 from urllib.parse import urlencode
 
@@ -29,11 +31,23 @@ LINKS_PER_PAGE = 5
 REQUESTS = 200
 BROAD_DEEP_PAGES = 10
 
-# Requests per crawl and delay of the benchmark that measures delayed requests,
+# Requests per crawl and delay of the benchmarks that wait, where wall time,
-# where wall time, unlike in the other benchmarks, is a function of the delay.
+# unlike in the other benchmarks, is a function of the delay.
 DELAYED_REQUESTS = 50
 DELAY = 0.005
 
+# Requests per crawl and items per response of the benchmarks that measure item
+# processing, which reaches fewer pages than the other benchmarks because every
+# page costs it several items.
+ITEM_REQUESTS = 20
+ITEMS_PER_RESPONSE = 100
+
+# Item concurrency limits of the benchmarks that measure item processing. The
+# high limit is above the number of items that a response yields in any of
+# them.
+HIGH_CONCURRENT_ITEMS = 1000
+DELAYED_CONCURRENT_ITEMS = 50
+
 NULL_SETTINGS: dict[str, Any] = {
     "DOWNLOAD_HANDLERS": {"http": NullDownloadHandler},
     "LOG_ENABLED": False,
@@ -63,7 +77,8 @@ class _FollowSpider(Spider):
 
 
 class _TreeSpider(Spider):
-    """Crawl *pages* pages on each of *domains* hostnames.
+    """Crawl *pages* pages on each of *domains* hostnames, yielding *items*
+    items from every page.
 
     Pages are numbered from 1, and page *n* links to pages *2n* and *2n+1*, so
     that requests also reach the scheduler from callbacks, and not only from
@@ -73,6 +88,7 @@ class _TreeSpider(Spider):
     name = "benchmark-tree"
     domains: int = 1
     pages: int = 1
+    items: int = 0
 
     async def start(self) -> AsyncIterator[Any]:
         for domain in range(self.domains):
@@ -83,6 +99,8 @@ class _TreeSpider(Spider):
         for child in (page * 2, page * 2 + 1):
             if child <= self.pages:
                 yield Request(response.urljoin(f"/{child}"))
+        for _ in range(self.items):
+            yield _Page(url=response.url)
 
 
 class _Pipeline:
@@ -90,12 +108,48 @@ class _Pipeline:
         return item
 
 
-def _crawl_tree(settings: dict[str, Any], *, domains: int, pages: int) -> Crawler:
+class _DelayedPipeline:
+    """Item pipeline that waits, so that the item concurrency limit applies.
+
+    The peak number of items of a same response in progress is tracked in the
+    ``benchmark/peak_items`` stat. Items are counted per response because the
+    limit is per response, and the items of a response are processed while
+    later responses are already being downloaded.
+    """
+
+    def __init__(self, crawler: Crawler):
+        self._crawler = crawler
+        self._active: Counter[str] = Counter()
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler) -> _DelayedPipeline:
+        return cls(crawler)
+
+    async def process_item(self, item: Any) -> Any:
+        url = item["url"]
+        self._active[url] += 1
+        assert self._crawler.stats
+        self._crawler.stats.max_value("benchmark/peak_items", self._active[url])
+        try:
+            await asyncio.sleep(DELAY)
+            return item
+        finally:
+            self._active[url] -= 1
+
+
+def _crawl_tree(
+    settings: dict[str, Any], *, domains: int, pages: int, items: int = 0
+) -> Crawler:
     crawler = crawl(
-        _TreeSpider, {**NULL_SETTINGS, **settings}, domains=domains, pages=pages
+        _TreeSpider,
+        {**NULL_SETTINGS, **settings},
+        domains=domains,
+        pages=pages,
+        items=items,
     )
     assert crawler.stats
     assert crawler.stats.get_value("downloader/response_count") == domains * pages
+    assert crawler.stats.get_value("item_scraped_count", 0) == domains * pages * items
     return crawler
 
 
@@ -163,3 +217,54 @@ def test_overhead_delay(benchmark: BenchmarkFixture) -> None:
     """
     settings = {"DOWNLOAD_DELAY": DELAY, "RANDOMIZE_DOWNLOAD_DELAY": False}
     benchmark(lambda: _crawl_tree(settings, domains=1, pages=DELAYED_REQUESTS))
+
+
+@pytest.mark.parametrize(
+    ("items", "settings"),
+    [
+        pytest.param(1, {}, id="single"),
+        pytest.param(ITEMS_PER_RESPONSE, {}, id="many"),
+        pytest.param(
+            1,
+            {"CONCURRENT_ITEMS": HIGH_CONCURRENT_ITEMS},
+            id="high-limit",
+        ),
+    ],
+)
+def test_overhead_items(
+    benchmark: BenchmarkFixture, items: int, settings: dict[str, Any]
+) -> None:
+    """Overhead of sending the items of a callback through the item pipeline.
+
+    The single and many scenarios, which use the default
+    :setting:`CONCURRENT_ITEMS` value, measure how that overhead grows with the
+    number of items that a response yields. The high-limit scenario instead
+    raises :setting:`CONCURRENT_ITEMS` well above that number.
+    """
+    benchmark(
+        lambda: _crawl_tree(settings, domains=1, pages=ITEM_REQUESTS, items=items)
+    )
+
+
+def test_overhead_item_concurrency(benchmark: BenchmarkFixture) -> None:
+    """Overhead of a crawl where item processing waits.
+
+    Every response yields more items than :setting:`CONCURRENT_ITEMS` allows in
+    parallel, so that the item pipeline gets them in several batches, and wall
+    time, unlike in most of the other benchmarks, is a function of the delay.
+    """
+    settings = {
+        "CONCURRENT_ITEMS": DELAYED_CONCURRENT_ITEMS,
+        "ITEM_PIPELINES": {_DelayedPipeline: 100},
+    }
+
+    def run() -> None:
+        crawler = _crawl_tree(
+            settings, domains=1, pages=ITEM_REQUESTS, items=ITEMS_PER_RESPONSE
+        )
+        assert crawler.stats
+        assert (
+            crawler.stats.get_value("benchmark/peak_items") == DELAYED_CONCURRENT_ITEMS
+        )
+
+    benchmark(run)

tests/benchmarks/test_urls.py

@@ -0,0 +1,152 @@
+from __future__ import annotations
+
+from html import escape
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+import pytest
+
+from scrapy import Request
+from scrapy.http import HtmlResponse
+from scrapy.linkextractors import LinkExtractor
+from scrapy.utils.request import fingerprint
+
+if TYPE_CHECKING:
+    from pytest_codspeed import BenchmarkFixture  # type: ignore[import-not-found]
+
+pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")
+
+RESPONSE_URL = "https://www.example.com/catalogue/page-1.html"
+
+# Links that each scenario returns for the benchmark page. They are fewer than
+# the anchors of the page because links to images, to other non-crawlable files
+# and to non-HTTP schemes are rejected, and, except in the scenario that keeps
+# duplicates, because the links that the navigation repeats are collapsed.
+LINKS = 63
+DUPLICATE_LINKS = 88
+CANONICAL_LINKS = 60
+FILTERED_LINKS = 45
+
+# Requests built from LINKS links that point to a different resource.
+# Canonicalization maps the rest to one that another link already covers, e.g.
+# two fragments of a page, or two spellings of one percent-escape.
+FINGERPRINTS = 60
+
+
+def _read_corpus() -> tuple[list[str], list[str]]:
+    """Return the URLs of ``urls.txt``, and its first group of URLs.
+
+    The first group is the site navigation, which the benchmark page repeats.
+    """
+    groups: list[list[str]] = [[]]
+    for line in (Path(__file__).parent / "urls.txt").read_text().splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            if groups[-1]:
+                groups.append([])
+            continue
+        groups[-1].append(line)
+    urls = [url for group in groups for url in group]
+    return urls, groups[0]
+
+
+def _build_page(urls: list[str], navigation: list[str]) -> bytes:
+    """Return an HTML page that links to *urls*.
+
+    Every link is surrounded by the markup of a product listing, so that
+    benchmarks also cover walking over the elements and attributes that a real
+    page puts between links.
+    """
+
+    def item(index: int, url: str) -> str:
+        href = escape(url)
+        return (
+            f'<li class="product" data-index="{index}">'
+            f'<img src="/media/thumbnail-{index}.jpg" alt="Product {index}" '
+            f'width="128" height="128">'
+            f'<h3><a href="{href}">Product {index}</a></h3>'
+            f'<p class="description">A description of product {index}.</p>'
+            f"</li>"
+        )
+
+    def nav(urls: list[str]) -> str:
+        links = "".join(f'<a href="{escape(url)}">{escape(url)}</a>' for url in urls)
+        return f'<nav class="site">{links}</nav>'
+
+    items = "".join(item(index, url) for index, url in enumerate(urls))
+    return (
+        "<!DOCTYPE html><html><head><title>Catalogue</title>"
+        f'<base href="{RESPONSE_URL}"></head><body>'
+        f'{nav(navigation)}<ul class="products">{items}</ul>{nav(navigation)}'
+        "</body></html>"
+    ).encode()
+
+
+URLS, NAVIGATION = _read_corpus()
+BODY = _build_page(URLS, NAVIGATION)
+
+
+def _response() -> HtmlResponse:
+    return HtmlResponse(RESPONSE_URL, body=BODY, encoding="utf-8")
+
+
+@pytest.mark.parametrize(
+    ("kwargs", "links"),
+    [
+        pytest.param({}, LINKS, id="default"),
+        pytest.param({"unique": False}, DUPLICATE_LINKS, id="duplicates"),
+        pytest.param({"canonicalize": True}, CANONICAL_LINKS, id="canonicalize"),
+        pytest.param(
+            {
+                "allow": r"/catalogue/",
+                "deny": r"/legal/",
+                "allow_domains": ["example.com", "www.example.com"],
+            },
+            FILTERED_LINKS,
+            id="filtered",
+        ),
+    ],
+)
+def test_extract_links(
+    benchmark: BenchmarkFixture, kwargs: dict[str, Any], links: int
+) -> None:
+    """Extraction of every link of a page.
+
+    The scenarios cover the choices that change which work dominates:
+    deduplication and canonicalization both build a key for every link, and the
+    filters of a configured extractor reject links before the later checks,
+    which the default extractor reaches for every link.
+    """
+    link_extractor = LinkExtractor(**kwargs)
+
+    def run() -> None:
+        assert len(link_extractor.extract_links(_response())) == links
+
+    benchmark(run)
+
+
+EXTRACTED_URLS = [link.url for link in LinkExtractor().extract_links(_response())]
+
+
+def test_requests(benchmark: BenchmarkFixture) -> None:
+    """Building a request for every link of a page."""
+
+    def run() -> None:
+        assert len([Request(url) for url in EXTRACTED_URLS]) == LINKS
+
+    benchmark(run)
+
+
+def test_fingerprints(benchmark: BenchmarkFixture) -> None:
+    """Fingerprinting the request of every link of a page.
+
+    Requests are built here as well, and not once for all rounds, because
+    fingerprints are cached per request object.
+    """
+
+    def run() -> None:
+        assert (
+            len({fingerprint(Request(url)) for url in EXTRACTED_URLS}) == FINGERPRINTS
+        )
+
+    benchmark(run)

tests/benchmarks/urls.txt

@@ -0,0 +1,130 @@
+# Link targets for the URL benchmarks, as they would appear in the href
+# attribute of a page at https://www.example.com/catalogue/page-1.html.
+#
+# Cost per URL varies by shape: the number of query parameters drives the
+# parsing and re-encoding of the query string, non-ASCII characters and
+# unescaped characters drive percent-encoding, and non-default ports, dot
+# segments and uppercase host names drive normalization. A corpus of uniform
+# URLs would therefore measure one shape and miss the others, so this one
+# covers each of them, in roughly the proportion of a real listing page.
+#
+# Blank lines and lines starting with "#" are ignored.
+
+# Site navigation. These also appear in a second copy of the navigation at the
+# end of the page, so that deduplication has duplicates to collapse.
+/
+/index.html
+/about-us
+/contact
+/catalogue/
+/catalogue/page-2.html
+/catalogue/page-3.html
+/help/faq
+/help/shipping-and-returns
+/legal/terms
+/legal/privacy
+
+# Relative paths of increasing depth.
+detail.html
+./detail.html
+../catalogue/page-4.html
+../../index.html
+/catalogue/category/books/fiction/index.html
+/catalogue/category/books/travel/mystery/historical/index.html
+/a/b/c/d/e/f/g/h/i/j/k/index.html
+
+# One query parameter.
+/catalogue/search?q=book
+/catalogue/page-1.html?page=2
+/catalogue/detail?id=1042
+
+# Several query parameters, in an order that canonicalization changes.
+/catalogue/search?q=book&sort=price
+/catalogue/search?sort=price&q=book
+/catalogue/search?q=book&sort=price&page=3&per_page=20&in_stock=1
+/catalogue/search?zone=eu&q=book&min=10&max=90&sort=rating&page=2&view=grid&lang=en&currency=EUR&ref=nav
+
+# Repeated keys, blank values and a bare key.
+/catalogue/search?tag=fiction&tag=travel&tag=history
+/catalogue/search?q=&sort=
+/catalogue/search?featured
+
+# Characters that need percent-encoding.
+/catalogue/search?q=cheap books
+/catalogue/detail/a book about books.html
+/catalogue/search?q=100%+cotton
+/catalogue/search?price=%3E10&title=A%20%26%20B
+
+# Percent-escapes that are already valid, in both cases.
+/catalogue/detail/%C3%A9dition-limit%C3%A9e.html
+/catalogue/detail/%c3%a9dition-limit%c3%a9e.html
+/catalogue/detail/%7Especial.html
+
+# Non-ASCII in the path and in the query.
+/catalogue/detail/édition-limitée.html
+/catalogue/search?q=édition
+/catalogue/búsqueda?q=libro&categoría=ficción
+/カタログ/詳細.html
+
+# Internationalized host names, encoded and decoded.
+https://例え.テスト/catalogue/page-1.html
+https://xn--r8jz45g.xn--zckzah/catalogue/page-2.html
+
+# Absolute URLs on the same host, on other hosts, and protocol-relative.
+https://www.example.com/catalogue/page-5.html
+https://www.example.com/catalogue/detail?id=1043
+http://www.example.com/catalogue/page-6.html
+https://shop.example.com/catalogue/page-1.html
+https://www.example.org/reviews/1042
+https://books.toscrape.com/catalogue/page-1.html
+//cdn.example.com/catalogue/page-7.html
+//www.example.com/catalogue/page-8.html
+
+# Ports, including the default one for the scheme.
+https://www.example.com:443/catalogue/page-9.html
+http://www.example.com:80/catalogue/page-10.html
+https://staging.example.com:8443/catalogue/page-1.html
+
+# Host name case, which normalization lowercases.
+https://WWW.EXAMPLE.COM/catalogue/Page-11.html
+HTTPS://www.example.com/catalogue/page-12.html
+
+# Dot segments, empty segments and trailing slashes, which WHATWG
+# normalization resolves and the standard library keeps.
+/catalogue/../catalogue/page-13.html
+/catalogue/./page-14.html
+/catalogue//page-15.html
+/catalogue/category/
+/catalogue/category
+
+# Fragments, which canonicalization drops and the deduplication key keeps.
+/catalogue/page-16.html#reviews
+/catalogue/page-16.html#description
+/catalogue/page-17.html#
+#top
+
+# Path parameters, where the semicolon is not the last segment.
+/catalogue;sessionid=abc123/page-18.html
+/catalogue/page-19.html;sessionid=abc123
+
+# User information in the authority.
+https://user:password@files.example.com/catalogue/page-1.html
+
+# A long URL, of the length that tracking parameters reach.
+/catalogue/search?q=book&utm_source=newsletter&utm_medium=email&utm_campaign=spring-sale-2026&utm_term=fiction%20paperback&utm_content=hero-banner-variant-b&session=6f1c9a2e4b7d8f0a1c3e5d7b9f2a4c6e&ref=https%3A%2F%2Fwww.example.org%2Freviews%2F1042&page=2&sort=relevance
+
+# Extensions that the default deny_extensions rejects, and one compound
+# extension, which only matches as a whole.
+/media/cover-1042.jpg
+/media/cover-1042.PNG
+/media/catalogue.pdf
+/static/style.css
+/static/app.js
+/downloads/catalogue.tar.gz
+/downloads/catalogue.zip
+
+# Schemes that are not crawlable, which are rejected before any parsing.
+mailto:orders@example.com
+javascript:void(0)
+tel:+441234567890
+data:text/plain,hello

tests/ignores.txt

@@ -1,3 +1,4 @@
 scrapy/core/downloader/handlers/http.py
 scrapy/extensions/statsmailer.py
+scrapy/interfaces.py
 scrapy/mail.py

tests/keys/__init__.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta, timezone
+from ipaddress import IPv4Address
 from pathlib import Path
 
 from cryptography.hazmat.backends import default_backend
@@ -12,6 +13,7 @@ from cryptography.hazmat.primitives.serialization import (
 from cryptography.x509 import (
     CertificateBuilder,
     DNSName,
+    IPAddress,
     Name,
     NameAttribute,
     SubjectAlternativeName,
@@ -53,7 +55,9 @@ def generate_keys():
         .not_valid_before(datetime.now(tz=timezone.utc))
         .not_valid_after(datetime.now(tz=timezone.utc) + timedelta(days=10))
         .add_extension(
-            SubjectAlternativeName([DNSName("localhost")]),
+            SubjectAlternativeName(
+                [DNSName("localhost"), IPAddress(IPv4Address("127.0.0.1"))]
+            ),
             critical=False,
         )
         .sign(key, SHA256(), default_backend())

tests/mockserver/ftp.py

@@ -10,7 +10,7 @@ from tempfile import mkdtemp
 from typing import TYPE_CHECKING
 
 from pyftpdlib.authorizers import DummyAuthorizer
-from pyftpdlib.handlers import FTPHandler
+from pyftpdlib.handlers import FTPHandler, TLS_FTPHandler
 from pyftpdlib.servers import FTPServer
 
 from tests.utils import get_script_run_env
@@ -25,27 +25,32 @@ if TYPE_CHECKING:
 class MockFTPServer:
     """Creates an FTP server on a random port with a default passwordless user
     (anonymous) and a temporary root path that you can read from the
-    :attr:`path` attribute."""
+    :attr:`path` attribute.
 
-    def __init__(self) -> None:
+    If *tls* is ``True``, the server requires FTPS, using the test certificate
-        self.proc: Popen[str] | None = None
+    from :file:`tests/keys`.
+    """
+
+    proc: Popen[str]
+    port: int
+    path: Path
+
+    def __init__(self, tls: bool = False) -> None:
         self.host: str = "127.0.0.1"
-        self.port: int | None = None
+        self.tls: bool = tls
-        self.path: Path | None = None
 
     def __enter__(self) -> Self:
         self.path = Path(mkdtemp())
         self.proc = Popen(
-            [sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)],
+            [sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)]
+            + (["--tls"] if self.tls else []),
             stderr=PIPE,
             env=get_script_run_env(),
             text=True,
         )
         assert self.proc.stderr is not None
         for line in self.proc.stderr:
-            if "starting FTP server" in line and (
+            if m := re.search(r"starting FTPS? .*on ([^ :]+):(\d+),", line):
-                m := re.search(r"starting FTP server on ([^ :]+):(\d+),", line)
-            ):
                 self.port = int(m.group(2))
                 break
         else:
@@ -63,23 +68,32 @@ class MockFTPServer:
         traceback: TracebackType | None,
     ) -> None:
         rmtree(str(self.path))
-        assert self.proc is not None
         self.proc.kill()
         self.proc.communicate()
 
     def url(self, path: str) -> str:
-        return f"ftp://{self.host}:{self.port}/{path}"
+        scheme = "ftps" if self.tls else "ftp"
+        return f"{scheme}://{self.host}:{self.port}/{path}"
 
 
 def main() -> None:
     parser = ArgumentParser()
     parser.add_argument("-d", "--directory", required=True)
+    parser.add_argument("--tls", action="store_true")
     args = parser.parse_args()
 
     authorizer = DummyAuthorizer()
     full_permissions = "elradfmwMT"
     authorizer.add_anonymous(args.directory, perm=full_permissions)
-    handler = FTPHandler
+    if args.tls:
+        keys = Path(__file__).parent.parent / "keys"
+        handler = TLS_FTPHandler
+        handler.certfile = str(keys / "localhost.crt")
+        handler.keyfile = str(keys / "localhost.key")
+        handler.tls_control_required = True
+        handler.tls_data_required = True
+    else:
+        handler = FTPHandler
     handler.authorizer = authorizer
     address = ("127.0.0.1", 0)
     server = FTPServer(address, handler)

tests/mockserver/http.py

@@ -11,6 +11,7 @@ from tests import tests_datadir
 from .http_base import BaseMockServer, main_factory
 from .http_resources import (
     ArbitraryLengthPayloadResource,
+    BadHeader,
     BaseResource,
     BrokenChunkedResource,
     BrokenDownloadResource,
@@ -52,6 +53,7 @@ class Root(BaseResource):
         put_child(self, b"partial", Partial())
         put_child(self, b"drop", Drop())
         put_child(self, b"raw", Raw())
+        put_child(self, b"bad-header", BadHeader())
         put_child(self, b"echo", Echo())
         put_child(self, b"payload", PayloadResource())
         put_child(self, b"alpayload", ArbitraryLengthPayloadResource())

tests/mockserver/http_resources.py

@@ -210,6 +210,39 @@ class Raw(LeafResource):
         request.finish()
 
 
+class BadHeader(LeafResource):
+    """Sends a response with a bad header line, one with no colon in it, like
+    some servers do, between two good ones.
+
+    One of the good header lines is split into two lines, so that handling of
+    such headers is also covered.
+    """
+
+    response = (
+        b"HTTP/1.1 200 OK\r\n"
+        b"Content-Length: 5\r\n"
+        b"Content-Type: text/html\r\n"
+        b"X-Folded-Header: one\r\n"
+        b"\ttwo\r\n"
+        b'<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />\r\n'
+        b"X-After-Bad-Header: works\r\n"
+        b"\r\n"
+        b"Works"
+    )
+
+    def render_GET(self, request: Request) -> int:
+        request.startedWriting = 1
+        self.deferRequest(request, 0, self._delayedRender, request)
+        return NOT_DONE_YET
+
+    def _delayedRender(self, request: Request) -> None:
+        request.write(self.response)
+        # Clients that stop parsing headers at the bad one don't get
+        # Content-Length, so they need the connection to be closed to know that
+        # the response body is over.
+        close_connection(request)
+
+
 class Echo(LeafResource):
     def render_GET(self, request: Request) -> bytes:
         assert request.content

tests/test_cmdline/__init__.py

@@ -1,3 +1,4 @@
+import ast
 import json
 import os
 import pstats
@@ -60,11 +61,8 @@ class TestCmdline:
             "-s",
             "EXTENSIONS=" + json.dumps(EXTENSIONS),
         )
-        # XXX: There's gotta be a smarter way to do this...
         assert "..." not in settingsstr
-        for char in ("'", "<", ">"):
+        settingsdict = ast.literal_eval(settingsstr)
-            settingsstr = settingsstr.replace(char, '"')
-        settingsdict = json.loads(settingsstr)
         assert set(settingsdict.keys()) == set(EXTENSIONS.keys())
         assert settingsdict[EXT_PATH] == 200