RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into py3.15

This commit is contained in:
Andrey Rakhmatullin 2026-06-11 12:15:25 +05:00
parent 72c5775faf 2d0a898e2d
commit ec2d79e1bb
112 changed files with 3373 additions and 2132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/checks.yml vendored
View File

@ -22,10 +22,10 @@ jobs:
TOXENV: pylint
- python-version: "3.10"
env:
TOXENV: typing
TOXENV: mypy
- python-version: "3.10"
env:
TOXENV: typing-tests
TOXENV: mypy-tests
# Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version.
- python-version: "3.14"
env:

16
.github/workflows/tests-ubuntu.yml vendored
View File

@ -45,26 +45,26 @@ jobs:
env:
TOXENV: pypy3
# pinned deps
# min deps
- python-version: "3.10.19"
env:
TOXENV: pinned
TOXENV: min
- python-version: "3.10.19"
env:
TOXENV: default-reactor-pinned
TOXENV: min-default-reactor
- python-version: "3.10.19"
env:
TOXENV: no-reactor-pinned
TOXENV: min-no-reactor
# pinned due to https://github.com/pypy/pypy/issues/5388
- python-version: pypy3.11-7.3.20
env:
TOXENV: pypy3-pinned
TOXENV: min-pypy3
- python-version: "3.10.19"
env:
TOXENV: extra-deps-pinned
TOXENV: min-extra-deps
- python-version: "3.10.19"
env:
TOXENV: botocore-pinned
TOXENV: min-botocore
- python-version: "3.14"
env:
@ -103,7 +103,7 @@ jobs:
python-version: ${{ matrix.python-version }}
- name: Install system libraries
if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') || contains(matrix.python-version, '3.15')
if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'min') || contains(matrix.python-version, '3.15')
run: |
sudo apt-get update
sudo apt-get install libxml2-dev libxslt-dev

6
.github/workflows/tests-windows.yml vendored
View File

@ -41,13 +41,13 @@ jobs:
env:
TOXENV: no-reactor
# pinned deps
# min deps
- python-version: "3.10.11"
env:
TOXENV: pinned
TOXENV: min
- python-version: "3.10.11"
env:
TOXENV: extra-deps-pinned
TOXENV: min-extra-deps
- python-version: "3.14"
env:

6
CITATION.cff Normal file
View File

@ -0,0 +1,6 @@
cff-version: 1.2.0
message: If you use Scrapy in published research, please cite it as below.
title: Scrapy
authors:
- name: Scrapy contributors
url: https://scrapy.org

4
SECURITY.md
View File

@ -4,8 +4,8 @@
| Version | Supported |
| ------- | ------------------ |
| 2.15.x | :white_check_mark: |
| < 2.15.x | :x: |
| 2.16.x | :white_check_mark: |
| < 2.16.x | :x: |
## Reporting a Vulnerability

21
conftest.py
View File

@ -24,13 +24,13 @@ def _py_files(folder):
collect_ignore = [
# may need extra deps
"docs/_ext",
# contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess
# contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerProcessSubprocess
*_py_files("tests/AsyncCrawlerProcess"),
# contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess
# contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerRunnerSubprocess
*_py_files("tests/AsyncCrawlerRunner"),
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
# contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerProcessSubprocess
*_py_files("tests/CrawlerProcess"),
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
# contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerRunnerSubprocess
*_py_files("tests/CrawlerRunner"),
]
@ -99,6 +99,19 @@ def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmPr
proxy.stop()
@pytest.fixture # function scope because it modifies os.environ
def socks5_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
proxy = MitmProxy(mode="socks5")
url = proxy.start()
monkeypatch.setenv("http_proxy", url)
monkeypatch.setenv("https_proxy", url)
try:
yield proxy
finally:
proxy.stop()
@pytest.fixture(scope="session")
def reactor_pytest(request) -> str:
return request.config.getoption("--reactor")

2
docs/_templates/layout.html vendored
View File

@ -1,6 +1,6 @@
{% extends "!layout.html" %}
{# Overriden to include a link to scrapy.org, not just to the docs root #}
{# Overridden to include a link to scrapy.org, not just to the docs root #}
{%- block sidebartitle %}
{# the logo helper function was removed in Sphinx 6 and deprecated since Sphinx 4 #}

17
docs/faq.rst
View File

@ -82,10 +82,18 @@ to steal from us!
Does Scrapy work with HTTP proxies?
-----------------------------------
Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP
Proxy downloader middleware. See
Yes. Support for HTTP proxies is provided through the HTTP Proxy downloader
middleware. See
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`.
Does Scrapy work with SOCKS proxies?
------------------------------------
Yes, when using
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. See
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and the
handler documentation.
How can I scrape an item with attributes in different pages?
------------------------------------------------------------
@ -360,7 +368,10 @@ method for this purpose. For example:
Does Scrapy support IPv6 addresses?
-----------------------------------
Yes, by setting :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
Yes, but when using
:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` or
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` you need to
set :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
Note that by doing so, you lose the ability to set a specific timeout for DNS requests
(the value of the :setting:`DNS_TIMEOUT` setting is ignored).

304
docs/news.rst
View File

@ -3,17 +3,284 @@
Release notes
=============
Scrapy VERSION (unreleased)
---------------------------
.. _release-2.16.0:
Scrapy 2.16.0 (2026-05-19)
--------------------------
Highlights:
- Official support for Python 3.14
- Support for Twisted 26.4.0+
Modified requirements
~~~~~~~~~~~~~~~~~~~~~
- Increased the minimum versions of the following dependencies:
- service_identity_: 18.1.0 → 23.1.0
(:issue:`7347`)
- Added support for Twisted 26.4.0+.
(:issue:`7347`, :issue:`7505`, :issue:`7520`)
- Added support for Python 3.14.
(:issue:`6604`, :issue:`7460`)
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` has been made
private as it's an implementation detail of
:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`.
- The following classes and functions, intended for internal use by
:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
and :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`, have
been made private:
- ``scrapy.core.downloader.handlers.http11.ScrapyAgent``
- ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent``
- ``scrapy.core.downloader.handlers.http11.TunnelingAgent``
- ``scrapy.core.downloader.handlers.http11.TunnelingTCP4ClientEndpoint``
- ``scrapy.core.downloader.handlers.http11.tunnel_request_data()``
- ``scrapy.core.downloader.handlers.http2.ScrapyH2Agent``
(:issue:`7496`, :issue:`7510`)
Deprecations
~~~~~~~~~~~~
- ``scrapy.FormRequest`` is deprecated. You can use the :doc:`form2request
<form2request:index>` library instead, see :ref:`form`.
(:issue:`6438`)
- ``scrapy.utils.python.MutableChain`` is deprecated.
(:issue:`7504`)
Deprecation removals
~~~~~~~~~~~~~~~~~~~~
- The ``start_requests()`` method of :class:`~scrapy.Spider`, deprecated in
2.13.0, is removed and no longer called. Use :meth:`~scrapy.Spider.start`
instead, or both to maintain support for lower Scrapy versions.
(:issue:`7490`)
- Support for ``process_start_requests()`` methods of :ref:`spider middlewares
<topics-spider-middleware>`, deprecated in 2.13.0, is removed. Use
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead,
or both to maintain support for lower Scrapy versions.
(:issue:`7490`)
- Support for synchronous ``process_spider_output()`` methods of spider
middlewares, deprecated in Scrapy 2.13.0, is removed. You should upgrade
the affected middlewares to have asynchronous ``process_spider_output()``
methods.
(:issue:`7504`)
- The ``spider`` arguments of the following methods of
:class:`~scrapy.core.scraper.Scraper`, deprecated in Scrapy 2.13.0, are
removed:
- ``close_spider()``
- ``enqueue_scrape()``
- ``handle_spider_error()``
- ``handle_spider_output()``
(:issue:`7487`)
- HTTP/1.0 support code, deprecated in Scrapy 2.13.0, is removed. This
includes:
- ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler``
- The ``scrapy.core.downloader.webclient`` module.
- The ``DOWNLOADER_HTTPCLIENTFACTORY`` setting.
(:issue:`7486`)
- The following functions, deprecated in Scrapy 2.13.0, are removed, you
should import them from :mod:`w3lib.url` directly instead:
- ``scrapy.utils.url.add_or_replace_parameter()``
- ``scrapy.utils.url.add_or_replace_parameters()``
- ``scrapy.utils.url.any_to_uri()``
- ``scrapy.utils.url.canonicalize_url()``
- ``scrapy.utils.url.file_uri_to_path()``
- ``scrapy.utils.url.is_url()``
- ``scrapy.utils.url.parse_data_uri()``
- ``scrapy.utils.url.parse_url()``
- ``scrapy.utils.url.path_to_file_uri()``
- ``scrapy.utils.url.safe_download_url()``
- ``scrapy.utils.url.safe_url_string()``
- ``scrapy.utils.url.url_query_cleaner()``
- ``scrapy.utils.url.url_query_parameter()``
(:issue:`7487`)
- The following test-related code, deprecated in Scrapy 2.13.0, is removed:
- the ``scrapy.utils.testproc`` module
- the ``scrapy.utils.testsite`` module
- ``scrapy.utils.test.assert_gcs_environ()``
- ``scrapy.utils.test.get_ftp_content_and_delete()``
- ``scrapy.utils.test.get_gcs_content_and_delete()``
- ``scrapy.utils.test.mock_google_cloud_storage()``
- ``scrapy.utils.test.skip_if_no_boto()``
- ``scrapy.utils.test.TestSpider``
(:issue:`7487`)
- ``scrapy.utils.versions.scrapy_components_versions()``, deprecated in
Scrapy 2.13.0, is removed, you can use
:func:`scrapy.utils.versions.get_versions` instead.
(:issue:`7487`)
- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` and
``scrapy.utils.url.escape_ajax()``, deprecated in Scrapy 2.13.0, are
removed.
(:issue:`7487`)
- The ``__init__()`` method of priority queue classes (see
:setting:`SCHEDULER_PRIORITY_QUEUE`) now needs to support a keyword-only
``start_queue_cls`` parameter, not supporting it was deprecated in Scrapy
2.13.0.
(:issue:`7487`)
- ``scrapy.spiders.init.InitSpider``, deprecated in Scrapy 2.13.0, is
removed.
(:issue:`7487`)
New features
~~~~~~~~~~~~
- New features and improvements for
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`:
- Support for proxies.
- Support for the :reqmeta:`download_latency` meta key.
- Support for :attr:`Response.certificate
<scrapy.http.Response.certificate>`.
- Default headers set by the ``httpx`` library are no longer added to
requests.
(:issue:`7441`, :issue:`7524`)
- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now
skips HTTPS proxy certificate verification when the
:setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting is set to ``False``.
(:issue:`7496`)
Improvements
~~~~~~~~~~~~
- :func:`time.monotonic` is used instead of :func:`time.time` to calculate
elapsed time in various places.
(:issue:`7377`)
- Improved extraction of the file extension from the URL in
:class:`~scrapy.pipelines.files.FilesPipeline`.
(:issue:`4225`, :issue:`7414`)
- Other code refactoring and improvements.
(:issue:`7401`)
Bug fixes
~~~~~~~~~
- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now
raises an exception when a request has an ``https://`` destination and an
``https://`` proxy, which is not supported by this handler. Previously it
tried to connect to the proxy via HTTP in this case.
(:issue:`7496`)
- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now
raises an exception for requests with ``http://`` URLs instead of trying to
connect, which is not supported by this handler.
(:issue:`7496`)
- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` no longer
adds the ``:status`` pseudo-header to :attr:`Response.headers
<scrapy.http.Response.headers>`.
(:issue:`7441`)
- Fixed :func:`scrapy.utils.response.open_in_browser` removing the ``<head>``
tag when adding the ``<base>`` tag.
(:issue:`7459`)
Documentation
~~~~~~~~~~~~~
- Documented that
:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
doesn't support HTTPS proxies for HTTPS destinations and that
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` doesn't
support proxies at all.
(:issue:`7496`)
- Added an example of using
:class:`logging.handlers.TimedRotatingFileHandler` to rotate Scrapy logs.
(:issue:`3628`, :issue:`7501`)
- Added a ``CITATION.cff`` file.
(:issue:`7502`, :issue:`7519`)
- Mentioned ``DOWNLOADER_CLIENT_TLS_METHOD`` in :ref:`bans`.
(:issue:`5232`, :issue:`7518`)
- Other documentation improvements and fixes.
(:issue:`7417`,
:issue:`7463`,
:issue:`7472`,
:issue:`7480`,
:issue:`7489`,
:issue:`7503`,
:issue:`7507`)
Quality assurance
~~~~~~~~~~~~~~~~~
- Added tests that connect to https://books.toscrape.com/ to test the
behavior with a real website. These tests are marked with the
``requires_internet`` pytest mark and can be skipped with e.g.
``-m 'not requires_internet'`` if you cannot or don't want to run them.
(:issue:`7520`)
- Type hints improvements and fixes.
(:issue:`7492`, :issue:`7532`)
- CI and test improvements and fixes.
(:issue:`7441`, :issue:`7466`, :issue:`7491`, :issue:`7496`)
.. _release-2.15.2:
Scrapy 2.15.2 (2026-04-28)
@ -1366,8 +1633,8 @@ Highlights:
- Added the :reqmeta:`allow_offsite` request meta key
- :ref:`Spider middlewares that don't support asynchronous spider output
<sync-async-spider-middleware>` are deprecated
- Spider middlewares that don't support asynchronous spider output are
deprecated
- Added a base class for :ref:`universal spider middlewares
<universal-spider-middleware>`
@ -1505,13 +1772,11 @@ Deprecations
``start_queue_cls`` parameter.
(:issue:`6752`)
- :ref:`Spider middlewares that don't support asynchronous spider output
<sync-async-spider-middleware>` are deprecated. The async iterable
downgrading feature, needed for using such middlewares with asynchronous
callbacks and with other spider middlewares that produce asynchronous
iterables, is also deprecated. Please update all such middlewares to
support asynchronous spider output.
(:issue:`6664`)
- Spider middlewares that don't support asynchronous spider output are
deprecated. The async iterable downgrading feature, needed for using such
middlewares with asynchronous callbacks and with other spider middlewares
that produce asynchronous iterables, is also deprecated. Please update all
such middlewares to support asynchronous spider output. (:issue:`6664`)
- Functions that were imported from :mod:`w3lib.url` and re-exported in
:mod:`scrapy.utils.url` are now deprecated, you should import them from
@ -1770,9 +2035,8 @@ Documentation
- Documented the setting values set in the default project template.
(:issue:`6762`, :issue:`6775`)
- Improved the :ref:`docs <sync-async-spider-middleware>` about asynchronous
iterable support in spider middlewares.
(:issue:`6688`)
- Improved the docs about asynchronous iterable support in spider
middlewares. (:issue:`6688`)
- Improved the :ref:`docs <coroutine-deferred-apis>` about using
:class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based
@ -1936,7 +2200,7 @@ Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- User-defined cookies for HTTPS requests will have the ``secure`` flag set
to ``True`` unless it's set to ``False`` explictly. This is important when
to ``True`` unless it's set to ``False`` explicitly. This is important when
these cookies are reused in HTTP requests, e.g. after a redirect to an HTTP
URL.
(:issue:`6357`)
@ -1971,7 +2235,7 @@ Backward-incompatible changes
``crawler.settings`` instead. When they call ``__init__()`` of the base
class they should pass the ``crawler`` argument to it too.
- A ``from_settings()`` method shouldn't be defined. Class-specific
initialization code should go into either an overriden ``from_crawler()``
initialization code should go into either an overridden ``from_crawler()``
method or into ``__init__()``.
- It's now possible to override ``from_crawler()`` and it's not necessary
to call ``MediaPipeline.from_crawler()`` in it if other recommendations
@ -7200,7 +7464,7 @@ This 1.1 release brings a lot of interesting features and bug fixes:
selectors engine without needing to upgrade Scrapy.
- HTTPS downloader now does TLS protocol negotiation by default,
instead of forcing TLS 1.0. You can also set the SSL/TLS method
using the new :setting:`DOWNLOADER_CLIENT_TLS_METHOD`.
using the new ``DOWNLOADER_CLIENT_TLS_METHOD`` setting.
- These bug fixes may require your attention:

146
docs/topics/coroutines.rst
View File

@ -23,9 +23,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :class:`~scrapy.Request` callbacks.
If you are using any custom or third-party :ref:`spider middleware
<topics-spider-middleware>`, see :ref:`sync-async-spider-middleware`.
- The :meth:`process_item` method of
:ref:`item pipelines <topics-item-pipeline>`.
@ -39,13 +36,9 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- The
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
method of :ref:`spider middlewares <topics-spider-middleware>`.
If defined as a coroutine, it must be an :term:`asynchronous generator`.
The input ``result`` parameter is an :term:`asynchronous iterable`.
See also :ref:`sync-async-spider-middleware` and
:ref:`universal-spider-middleware`.
method of :ref:`spider middlewares <topics-spider-middleware>`, which
*must* be defined as an :term:`asynchronous generator` except in
:ref:`universal spider middlewares <universal-spider-middleware>`.
- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method
of :ref:`spider middlewares <custom-spider-middleware>`, which *must* be
@ -277,136 +270,3 @@ You can also send multiple requests in parallel:
"price": responses[0][1].css(".price::text").get(),
"price2": responses[1][1].css(".color::text").get(),
}
.. _sync-async-spider-middleware:
Mixing synchronous and asynchronous spider middlewares
======================================================
The output of a :class:`~scrapy.Request` callback is passed as the ``result``
parameter to the
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
of the first :ref:`spider middleware <topics-spider-middleware>` from the
:ref:`list of active spider middlewares <topics-spider-middleware-setting>`.
Then the output of that ``process_spider_output`` method is passed to the
``process_spider_output`` method of the next spider middleware, and so on for
every active spider middleware.
Scrapy supports mixing :ref:`coroutine methods <async>` and synchronous methods
in this chain of calls.
However, if any of the ``process_spider_output`` methods is defined as a
synchronous method, and the previous ``Request`` callback or
``process_spider_output`` method is a coroutine, there are some drawbacks to
the asynchronous-to-synchronous conversion that Scrapy does so that the
synchronous ``process_spider_output`` method gets a synchronous iterable as its
``result`` parameter:
- The whole output of the previous ``Request`` callback or
``process_spider_output`` method is awaited at this point.
- If an exception raises while awaiting the output of the previous
``Request`` callback or ``process_spider_output`` method, none of that
output will be processed.
This contrasts with the regular behavior, where all items yielded before
an exception raises are processed.
Asynchronous-to-synchronous conversions are supported for backward
compatibility, but they are deprecated and will stop working in a future
version of Scrapy.
To avoid asynchronous-to-synchronous conversions, when defining ``Request``
callbacks as coroutine methods or when using spider middlewares whose
``process_spider_output`` method is an :term:`asynchronous generator`, all
active spider middlewares must either have their ``process_spider_output``
method defined as an asynchronous generator or :ref:`define a
process_spider_output_async method <universal-spider-middleware>`.
.. _sync-async-spider-middleware-users:
For middleware users
--------------------
If you have asynchronous callbacks or use asynchronous-only spider middlewares
you should make sure the asynchronous-to-synchronous conversions
:ref:`described above <sync-async-spider-middleware>` don't happen. To do this,
make sure all spider middlewares you use support asynchronous spider output.
Even if you don't have asynchronous callbacks and don't use asynchronous-only
spider middlewares in your project, it's still a good idea to make sure all
middlewares you use support asynchronous spider output, so that it will be easy
to start using asynchronous callbacks in the future. Because of this, Scrapy
logs a warning when it detects a synchronous-only spider middleware.
If you want to update middlewares you wrote, see the :ref:`following section
<sync-async-spider-middleware-authors>`. If you have 3rd-party middlewares that
aren't yet updated by their authors, you can :ref:`subclass <tut-inheritance>`
them to make them :ref:`universal <universal-spider-middleware>` and use the
subclasses in your projects.
.. _sync-async-spider-middleware-authors:
For middleware authors
----------------------
If you have a spider middleware that defines a synchronous
``process_spider_output`` method, you should update it to support asynchronous
spider output for :ref:`better compatibility <sync-async-spider-middleware>`,
even if you don't yet use it with asynchronous callbacks, especially if you
publish this middleware for other people to use. You have two options for this:
1. Make the middleware asynchronous, by making the ``process_spider_output``
method an :term:`asynchronous generator`.
2. Make the middleware universal, as described in the :ref:`next section
<universal-spider-middleware>`.
If your middleware won't be used in projects with synchronous-only middlewares,
e.g. because it's an internal middleware and you know that all other
middlewares in your projects are already updated, it's safe to choose the first
option. Otherwise, it's better to choose the second option.
.. _universal-spider-middleware:
Universal spider middlewares
----------------------------
To allow writing a spider middleware that supports asynchronous execution of
its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding
:ref:`asynchronous-to-synchronous conversions <sync-async-spider-middleware>`)
while maintaining support for older Scrapy versions, you may define
``process_spider_output`` as a synchronous method and define an
:term:`asynchronous generator` version of that method with an alternative name:
``process_spider_output_async``.
For example:
.. code-block:: python
class UniversalSpiderMiddleware:
def process_spider_output(self, response, result):
for r in result:
# ... do something with r
yield r
async def process_spider_output_async(self, response, result):
async for r in result:
# ... do something with r
yield r
.. note:: This is an interim measure to allow, for a time, to write code that
works in Scrapy 2.7 and later without requiring
asynchronous-to-synchronous conversions, and works in earlier Scrapy
versions as well.
In some future version of Scrapy, however, this feature will be
deprecated and, eventually, in a later version of Scrapy, this
feature will be removed, and all spider middlewares will be expected
to define their ``process_spider_output`` method as an asynchronous
generator.
Since 2.13.0, Scrapy provides a base class,
:class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which implements
the ``process_spider_output()`` and ``process_spider_output_async()`` methods,
so instead of duplicating the processing code you can override the
``get_processed_request()`` and/or the ``get_processed_item()`` method.

220
docs/topics/download-handlers.rst
View File

@ -130,44 +130,39 @@ these exceptions.
.. _download-handlers-ref:
Built-in download handlers reference
====================================
Built-in HTTP download handlers reference
=========================================
DataURIDownloadHandler
----------------------
Scrapy ships several handlers for HTTP and HTTPS requests. While all of them
support basic features, they may differ in support of specific Scrapy features
and settings and HTTP protocol features. See the documentation of specific
handlers and specific settings for more information. Additionally, as the
underlying HTTP client implementations differ between handlers, the behavior of
specific websites may be different when doing the same Scrapy requests but
using different handlers.
.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler
Here is a comparison of some features of the built-in HTTP handlers, see the
individual handler docs for more differences:
| Supported scheme: ``data``.
| Lazy: no.
================== ================= ===================== ====================
Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler
================== ================= ===================== ====================
Requires asyncio No No Yes
Requires a reactor Yes Yes No
HTTP/1.1 No Yes Yes
HTTP/2 Yes No Yes
TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl``
HTTP proxies No Yes Yes
SOCKS proxies No No Yes
================== ================= ===================== ====================
This handler supports RFC 2397 ``data:content/type;base64,`` data URIs.
You can find additional HTTP download handlers in the
scrapy-download-handlers-incubator_ package. This package is made by the Scrapy
developers and contains experimental handlers that may be included in some
later Scrapy version but can already be used. Please refer to the documentation
of this package for more information.
FileDownloadHandler
-------------------
.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler
| Supported scheme: ``file``.
| Lazy: no.
This handler supports ``file:///path`` local file URIs. It doesn't
support remote files.
FTPDownloadHandler
------------------
.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler
| Supported scheme: ``ftp``.
| Lazy: no.
This handler supports ``ftp://host/path`` FTP URIs.
It's implemented using :mod:`twisted.protocols.ftp`.
.. note::
This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
.. _scrapy-download-handlers-incubator: https://github.com/scrapy-plugins/scrapy-download-handlers-incubator
.. _twisted-http2-handler:
@ -177,7 +172,9 @@ H2DownloadHandler
.. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler
| Supported scheme: ``https``.
| Lazy: yes.
| :ref:`Lazy <lazy-download-handlers>`: yes.
| :ref:`Requires asyncio support <using-asyncio>`: no.
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.
This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocol
for them.
@ -196,63 +193,96 @@ If you want to use this handler you need to replace the default one for the
"https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",
}
Features and limitations
^^^^^^^^^^^^^^^^^^^^^^^^
.. warning::
This handler is experimental, and not yet recommended for production
environments. Future Scrapy versions may introduce related changes without
a deprecation period or warning.
.. note::
=========================== ================================================
HTTP proxies No (not implemented)
SOCKS proxies No (not supported by the library)
HTTP/2 Yes
``response.certificate`` :class:`twisted.internet.ssl.Certificate` object
Per-request ``bindaddress`` Yes
TLS implementation ``pyOpenSSL``/``cryptography``
=========================== ================================================
Known limitations of the HTTP/2 implementation in this handler include:
Other limitations:
- No support for proxies.
- No support for HTTP/1.1.
- No support for HTTP/2 Cleartext (h2c), since no major browser supports
HTTP/2 unencrypted (refer `http2 faq`_).
- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`
to ``scrapy.resolver.CachingHostnameResolver``.
- No setting to specify a maximum `frame size`_ larger than the default
value, 16384. Connections to servers that send a larger frame will
fail.
- No support for the :signal:`bytes_received` and :signal:`headers_received`
signals.
- No support for `server pushes`_, which are ignored.
Known limitations of the HTTP/2 support:
- No support for the :signal:`bytes_received` and
:signal:`headers_received` signals.
- No support for HTTP/2 Cleartext (h2c), since no major browser supports
HTTP/2 unencrypted (refer `http2 faq`_).
- No setting to specify a maximum `frame size`_ larger than the default
value, 16384. Connections to servers that send a larger frame will fail.
- No support for `server pushes`_, which are ignored.
.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2
.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2
.. note::
This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
HTTP11DownloadHandler
---------------------
.. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler
| Supported schemes: ``http``, ``https``.
| Lazy: no.
| :ref:`Lazy <lazy-download-handlers>`: no.
| :ref:`Requires asyncio support <using-asyncio>`: no.
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.
This handler supports ``http://host/path`` and ``https://host/path`` URLs and
uses the HTTP/1.1 protocol for them.
It's implemented using :mod:`twisted.web.client`.
.. note::
This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
Features and limitations
^^^^^^^^^^^^^^^^^^^^^^^^
=========================== ================================================
HTTP proxies Yes
SOCKS proxies No (not supported by the library)
HTTP/2 No (implemented as a separate handler)
``response.certificate`` :class:`twisted.internet.ssl.Certificate` object
Per-request ``bindaddress`` Yes
TLS implementation ``pyOpenSSL``/``cryptography``
=========================== ================================================
Other limitations:
- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`
to ``scrapy.resolver.CachingHostnameResolver``.
- HTTPS proxies to HTTPS destinations are not supported.
HttpxDownloadHandler
--------------------
.. versionadded:: 2.15.0
.. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler
| Supported schemes: ``http``, ``https``.
| Lazy: no.
| :ref:`Lazy <lazy-download-handlers>`: no.
| :ref:`Requires asyncio support <using-asyncio>`: yes.
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
This handler supports ``http://host/path`` and ``https://host/path`` URLs and
uses the HTTP/1.1 protocol for them.
uses the HTTP/1.1 or HTTP/2 protocol for them.
It's implemented using the ``httpx`` library and needs it to be installed.
@ -266,33 +296,83 @@ If you want to use this handler you need to replace the default ones for the
"https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
}
Features and limitations
^^^^^^^^^^^^^^^^^^^^^^^^
.. warning::
This handler is experimental, and not yet recommended for production
environments. Future Scrapy versions may introduce related changes without
a deprecation period or warning or even remove it altogether.
.. note::
=========================== =======================================
HTTP proxies Yes
SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``)
HTTP/2 Yes (requires ``httpx[http2]``)
``response.certificate`` DER bytes
Per-request ``bindaddress`` No (not supported by the library)
TLS implementation Standard library ``ssl``
=========================== =======================================
As this handler is based on a different HTTP client implementation compared
to :class:`~.HTTP11DownloadHandler`, it's expected that its behavior on
some websites may be different. Additionally, these are the Scrapy features
that are explicitly not supported when using it:
Other limitations:
- Proxy support (the :reqmeta:`proxy` meta key).
- The handler creates a separate connection pool for each proxy URL (due to
limitations of ``httpx``) which may lead to higher resource usage when
using proxy rotation.
- Per-request bind address support (the :reqmeta:`bindaddress` meta key).
The global :setting:`DOWNLOAD_BIND_ADDRESS` setting is supported but the
port number, if specified, will be ignored.
.. setting:: HTTPX_HTTP2_ENABLED
- The :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` and
:setting:`DOWNLOADER_CLIENT_TLS_METHOD` settings.
HTTPX_HTTP2_ENABLED
^^^^^^^^^^^^^^^^^^^
- Settings specific to the Twisted networking or HTTP implementation, like
:setting:`DNS_RESOLVER`.
Default: ``False``
- Using :ref:`non-asyncio reactors <disable-asyncio>` (``httpx`` requires
``asyncio``).
Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra
needs to be installed if you want to enable this setting.
.. versionadded:: VERSION
Built-in non-HTTP download handlers reference
=============================================
DataURIDownloadHandler
----------------------
.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler
| Supported scheme: ``data``.
| :ref:`Lazy <lazy-download-handlers>`: no.
| :ref:`Requires asyncio support <using-asyncio>`: no.
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
This handler supports RFC 2397 ``data:content/type;base64,`` data URIs.
FileDownloadHandler
-------------------
.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler
| Supported scheme: ``file``.
| :ref:`Lazy <lazy-download-handlers>`: no.
| :ref:`Requires asyncio support <using-asyncio>`: no.
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
This handler supports ``file:///path`` local file URIs. It doesn't
support remote files.
FTPDownloadHandler
------------------
.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler
| Supported scheme: ``ftp``.
| :ref:`Lazy <lazy-download-handlers>`: no.
| :ref:`Requires asyncio support <using-asyncio>`: no.
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: yes.
This handler supports ``ftp://host/path`` FTP URIs.
It's implemented using :mod:`twisted.protocols.ftp`.
S3DownloadHandler
-----------------
@ -300,7 +380,9 @@ S3DownloadHandler
.. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler
| Supported scheme: ``s3``.
| Lazy: yes.
| :ref:`Lazy <lazy-download-handlers>`: yes.
| :ref:`Requires asyncio support <using-asyncio>`: no.
| :ref:`Requires a Twisted reactor <asyncio-without-reactor>`: no.
This handler supports ``s3://bucket/path`` S3 URIs.

25
docs/topics/downloader-middleware.rst
View File

@ -745,8 +745,7 @@ HttpProxyMiddleware
Handling of this meta key needs to be implemented inside the :ref:`download
handler <topics-download-handlers>`, so it's not guaranteed to be supported
by all 3rd-party handlers. It's currently unsupported by
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` and
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`.
.. note::
@ -758,6 +757,13 @@ HttpProxyMiddleware
:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
supports HTTPS proxies only for HTTP destinations.
.. note::
If the download handler supports it, you can use a SOCKS proxy URL (e.g.
``socks5://username:password@some_proxy_server:port``).
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`
supports SOCKS proxies while other built-in handlers don't.
HttpProxyMiddleware settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@ -1053,6 +1059,21 @@ has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught
exception propagation, see
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`.
.. setting:: RETRY_GIVE_UP_LOG_LEVEL
RETRY_GIVE_UP_LOG_LEVEL
^^^^^^^^^^^^^^^^^^^^^^^
Default: ``"ERROR"``
:ref:`Logging level <levels>` used for the message logged when a request
exceeds its retries.
Can be a level name (e.g. ``"WARNING"``) or a number (e.g. ``logging.WARNING``
or ``30``).
See also: :reqmeta:`give_up_log_level`, :func:`get_retry_request`.
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST

15
docs/topics/exporters.rst
View File

@ -93,24 +93,25 @@ described next.
1. Declaring a serializer in the field
--------------------------------------
If you use :class:`~scrapy.Item` you can declare a serializer in the
:ref:`field metadata <topics-items-fields>`. The serializer must be
a callable which receives a value and returns its serialized form.
Every :ref:`item type <item-types>` except :class:`dict` lets you declare a
serializer in the :ref:`field metadata <topics-items-fields>`. The serializer
must be a callable which receives a value and returns its serialized form.
Example:
.. code-block:: python
import scrapy
from dataclasses import dataclass, field
def serialize_price(value):
return f"$ {str(value)}"
class Product(scrapy.Item):
name = scrapy.Field()
price = scrapy.Field(serializer=serialize_price)
@dataclass
class Product:
name: str
price: float = field(metadata={"serializer": serialize_price})
2. Overriding the serialize_field() method

23
docs/topics/extensions.rst
View File

@ -136,6 +136,19 @@ Core Stats extension
Enable the collection of core statistics, provided the stats collection is
enabled (see :ref:`topics-stats`).
The following stats are collected:
* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`).
* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`).
* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`).
* ``finish_reason``: the closing reason string (e.g. ``"finished"``,
``"closespider_timeout"``).
* ``item_scraped_count``: total number of items that passed all pipelines.
* ``item_dropped_count``: total number of items dropped by a pipeline.
* ``item_dropped_reasons_count/<ExceptionName>``: per-exception drop count
(e.g. ``item_dropped_reasons_count/DropItem``).
* ``response_received_count``: total number of HTTP responses received.
.. _topics-extensions-ref-telnetconsole:
Log Count extension
@ -247,6 +260,7 @@ settings:
* :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`
* :setting:`CLOSESPIDER_ITEMCOUNT`
* :setting:`CLOSESPIDER_PAGECOUNT`
* :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM`
* :setting:`CLOSESPIDER_ERRORCOUNT`
.. note::
@ -260,12 +274,11 @@ settings:
CLOSESPIDER_TIMEOUT
"""""""""""""""""""
Default: ``0``
Default: ``0.0``
An integer which specifies a number of seconds. If the spider remains open for
more than that number of seconds, it will be automatically closed with the
reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by
timeout.
If the spider remains open for more than this number of seconds, it will be
automatically closed with the reason ``closespider_timeout``. If zero (or non
set), spiders won't be closed by timeout.
.. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM

31
docs/topics/loaders.rst
View File

@ -102,14 +102,13 @@ One approach to overcome this is to define items using the
.. code-block:: python
from dataclasses import dataclass, field
from typing import Optional
@dataclass
class InventoryItem:
name: Optional[str] = field(default=None)
price: Optional[float] = field(default=None)
stock: Optional[int] = field(default=None)
name: str | None = field(default=None)
price: float | None = field(default=None)
stock: int | None = field(default=None)
.. _topics-loaders-processors:
@ -228,7 +227,8 @@ metadata. Here is an example:
.. code-block:: python
import scrapy
from dataclasses import dataclass, field
from itemloaders.processors import Join, MapCompose, TakeFirst
from w3lib.html import remove_tags
@ -238,14 +238,21 @@ metadata. Here is an example:
return value
class Product(scrapy.Item):
name = scrapy.Field(
input_processor=MapCompose(remove_tags),
output_processor=Join(),
@dataclass
class Product:
name: str | None = field(
default=None,
metadata={
"input_processor": MapCompose(remove_tags),
"output_processor": Join(),
},
)
price = scrapy.Field(
input_processor=MapCompose(remove_tags, filter_price),
output_processor=TakeFirst(),
price: str | None = field(
default=None,
metadata={
"input_processor": MapCompose(remove_tags, filter_price),
"output_processor": TakeFirst(),
},
)

16
docs/topics/media-pipeline.rst
View File

@ -81,9 +81,6 @@ thumbnailing and normalizing images to JPEG/RGB format.
Enabling your Media Pipeline
============================
.. setting:: IMAGES_STORE
.. setting:: FILES_STORE
To enable your media pipeline you must first add it to your project
:setting:`ITEM_PIPELINES` setting.
@ -102,6 +99,8 @@ For Files Pipeline, use:
.. note::
You can also use both the Files and Images Pipeline at the same time.
.. setting:: IMAGES_STORE
.. setting:: FILES_STORE
Then, configure the target storage setting to a valid value that will be used
for storing the downloaded images. Otherwise the pipeline will remain disabled,
@ -337,17 +336,18 @@ respectively), the pipeline will put the results under the respective field
When using :ref:`item types <item-types>` for which fields are defined beforehand,
you must define both the URLs field and the results field. For example, when
using the images pipeline, items must define both the ``image_urls`` and the
``images`` field. For instance, using the :class:`~scrapy.Item` class:
``images`` field. For instance, using a dataclass:
.. code-block:: python
import scrapy
from dataclasses import dataclass, field
class MyItem(scrapy.Item):
@dataclass
class MyItem:
# ... other item fields ...
image_urls = scrapy.Field()
images = scrapy.Field()
image_urls: list[str] = field(default_factory=list)
images: list[dict] = field(default_factory=list)
If you want to use another field name for the URLs key or for the results key,
it is also possible to override it.

4
docs/topics/practices.rst
View File

@ -409,6 +409,10 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
super proxy that you can attach your own proxies to.
* for HTTPS websites, if blocking appears related to TLS behavior, consider
adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and
:setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond
differently depending on the TLS method used by the client.
* use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy
plugin <https://github.com/scrapy-plugins/scrapy-zyte-api>`__ and additional
features, like `AI web scraping <https://www.zyte.com/ai-web-scraping/>`__

51
docs/topics/request-response.rst
View File

@ -117,6 +117,9 @@ Request objects
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
This encoding will be used to percent-encode the URL and to convert the
body to bytes (if given as a string).
To disable URL percent-encoding for a request, use the
:reqmeta:`verbatim_url` request meta key.
:type encoding: str
:param priority: sets :attr:`priority`, defaults to ``0``.
@ -136,9 +139,13 @@ Request objects
.. attribute:: Request.url
A string containing the URL of this request. Keep in mind that this
attribute contains the escaped URL, so it can differ from the URL passed in
the ``__init__()`` method.
A string containing the URL of this request.
Keep in mind that this attribute contains the escaped URL, so it can
differ from the URL passed in the ``__init__()`` method.
If :reqmeta:`verbatim_url` is set to ``True``, the URL is kept as
passed to ``__init__()``.
This attribute is read-only. To change the URL of a Request use
:meth:`replace`.
@ -541,6 +548,11 @@ in your :meth:`fingerprint` method implementation:
.. autofunction:: scrapy.utils.request.fingerprint
By default, request fingerprinting canonicalizes the request URL. If
:reqmeta:`verbatim_url` is set to ``True``, fingerprinting does not
canonicalize the URL, and the ``keep_fragments`` parameter is ignored (it is
effectively true).
For example, to take the value of a request header named ``X-ID`` into
account:
@ -702,6 +714,7 @@ Those are:
* :reqmeta:`download_timeout`
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
* ``ftp_user`` (See :setting:`FTP_USER` for more info)
* :reqmeta:`give_up_log_level`
* :reqmeta:`handle_httpstatus_all`
* :reqmeta:`handle_httpstatus_list`
* :reqmeta:`is_start_request`
@ -710,6 +723,7 @@ Those are:
* :reqmeta:`redirect_reasons`
* :reqmeta:`redirect_urls`
* :reqmeta:`referrer_policy`
* :reqmeta:`verbatim_url`
.. reqmeta:: bindaddress
@ -777,15 +791,38 @@ download_fail_on_dataloss
Whether or not to fail on broken responses. See:
:setting:`DOWNLOAD_FAIL_ON_DATALOSS`.
.. reqmeta:: give_up_log_level
give_up_log_level
-----------------
:ref:`Logging level <levels>` used for the message logged when a request
exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
.. reqmeta:: max_retry_times
max_retry_times
---------------
The meta key is used set retry times per request. When initialized, the
The meta key is used set retry times per request. When set, the
:reqmeta:`max_retry_times` meta key takes higher precedence over the
:setting:`RETRY_TIMES` setting.
.. reqmeta:: verbatim_url
verbatim_url
------------
Set this key to ``True`` to keep the request URL as passed to
:class:`~scrapy.Request`, without URL percent-encoding.
When this key is enabled, :func:`~scrapy.utils.request.fingerprint` does not
canonicalize the request URL, so requests whose URLs differ only in
characters that would otherwise be canonicalized get different fingerprints.
In this mode, the ``keep_fragments`` parameter is ignored, and it is
effectively true.
.. _topics-stop-response-download:
@ -917,7 +954,7 @@ Response objects
:type request: scrapy.Request
:param certificate: an object representing the server's SSL certificate.
:type certificate: twisted.internet.ssl.Certificate
:type certificate: typing.Any
:param ip_address: The IP address of the server from which the Response originated.
:type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
@ -1009,8 +1046,8 @@ Response objects
.. attribute:: Response.certificate
A :class:`twisted.internet.ssl.Certificate` object representing
the server's SSL certificate.
An object representing the server's SSL certificate. Its type and
contents depend on the download handler that produced the response.
Only populated for ``https`` responses, ``None`` otherwise.

50
docs/topics/settings.rst
View File

@ -737,32 +737,50 @@ specific cipher that is not included in ``DEFAULT`` if a website requires it.
by all 3rd-party handlers. It's currently unsupported by
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
.. setting:: DOWNLOADER_CLIENT_TLS_METHOD
.. setting:: DOWNLOAD_TLS_MAX_VERSION
DOWNLOADER_CLIENT_TLS_METHOD
----------------------------
DOWNLOAD_TLS_MAX_VERSION
------------------------
Default: ``'TLS'``
Default: ``None``
Use this setting to customize the TLS/SSL method used by the HTTPS download
handler.
Use this setting to change the maximum version of the TLS protocol allowed to
be used by Scrapy.
This setting must be one of these string values:
This setting must be either ``None``, in which case it doesn't affect the
version selection, or one of these string values:
- ``'TLS'``: maps to OpenSSL's ``TLS_method()`` (a.k.a ``SSLv23_method()``),
which allows protocol negotiation, starting from the highest supported
by the platform; **default, recommended**
- ``'TLSv1.0'``: this value forces HTTPS connections to use TLS version 1.0 ;
set this if you want the behavior of Scrapy<1.1
- ``'TLSv1.1'``: forces TLS version 1.1
- ``'TLSv1.2'``: forces TLS version 1.2
- ``'TLSv1.0'``
- ``'TLSv1.1'``
- ``'TLSv1.2'``
- ``'TLSv1.3'``
The range of allowed TLS versions advertised by Scrapy when making TLS
connections will depend on the TLS implementation defaults and the values of
:setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION`.
It's possible to re-enable versions that are supported by the TLS
implementation but disabled by default by adjusting these settings, but it's
impossible to enable unsupported ones, such as any versions below 1.2 in many
modern environments.
.. note::
Handling of this setting needs to be implemented inside the :ref:`download
handler <topics-download-handlers>`, so it's not guaranteed to be supported
by all 3rd-party handlers. It's currently unsupported by
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
by all 3rd-party handlers. Additionally, the set of supported TLS versions
depends on the TLS implementation being used by the handler.
.. setting:: DOWNLOAD_TLS_MIN_VERSION
DOWNLOAD_TLS_MIN_VERSION
------------------------
Default: ``None``
Use this setting to change the minimum version of the TLS protocol allowed to
be used by Scrapy.
See :setting:`DOWNLOAD_TLS_MAX_VERSION` for the details and limitations.
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING

63
docs/topics/spider-middleware.rst
View File

@ -117,36 +117,28 @@ one or more of these methods:
:type response: :class:`~scrapy.http.Response` object
.. method:: process_spider_output(response, result)
:async:
This method is called with the results returned from the Spider, after
it has processed the response.
This method is an :term:`asynchronous generator` called with the
results from the spider after the spider has processed the response.
:meth:`process_spider_output` must return an iterable of
:class:`~scrapy.Request` objects and :ref:`item objects
<topics-items>`.
Consider defining this method as an :term:`asynchronous generator`,
which will be a requirement in a future version of Scrapy. However, if
you plan on sharing your spider middleware with other people, consider
either :ref:`enforcing Scrapy 2.7 <enforce-component-requirements>`
as a minimum requirement of your spider middleware, or :ref:`making
your spider middleware universal <universal-spider-middleware>` so that
it works with Scrapy versions earlier than Scrapy 2.7.
.. seealso:: :ref:`universal-spider-middleware`.
:param response: the response which generated this output from the
spider
:type response: :class:`~scrapy.http.Response` object
:param result: the result returned by the spider
:type result: an iterable of :class:`~scrapy.Request` objects and
:ref:`item objects <topics-items>`
:param result: the results from the spider
:type result: an :term:`asynchronous iterable` of
:class:`~scrapy.Request` objects and :ref:`item objects
<topics-items>`
.. method:: process_spider_output_async(response, result)
:async:
If defined, this method must be an :term:`asynchronous generator`,
which will be called instead of :meth:`process_spider_output` if
``result`` is an :term:`asynchronous iterable`.
Alternative name for :meth:`process_spider_output` used when
implementing a :ref:`universal spider middleware
<universal-spider-middleware>`.
.. method:: process_spider_exception(response, exception)
@ -174,13 +166,40 @@ one or more of these methods:
:type exception: :exc:`Exception` object
.. _universal-spider-middleware:
Universal spider middlewares
----------------------------
In Scrapy 2.6.3 and lower, ``process_spider_output()`` must be a *synchronous*
generator.
To support those versions and higher Scrapy versions in the same middleware,
rename your asynchronous :meth:`~SpiderMiddleware.process_spider_output`
method to :meth:`~SpiderMiddleware.process_spider_output_async`, and define a
synchronous ``process_spider_output()`` method to be used by 2.6.3 and lower
versions.
For example:
.. code-block:: python
class UniversalSpiderMiddleware:
async def process_spider_output_async(self, response, result):
async for r in result:
# ... do something with r
yield r
def process_spider_output(self, response, result):
for r in result:
# ... do something with r
yield r
Base class for custom spider middlewares
----------------------------------------
Scrapy provides a base class for custom spider middlewares. It's not required
to use it but it can help with simplifying middleware implementations and
reducing the amount of boilerplate code in :ref:`universal middlewares
<universal-spider-middleware>`.
to use it but it can help with simplifying middleware implementations.
.. module:: scrapy.spidermiddlewares.base

26
docs/topics/spiders.rst
View File

@ -457,13 +457,14 @@ with a ``TestItem`` declared in a ``myproject.items`` module:
.. code-block:: python
import scrapy
from dataclasses import dataclass
class TestItem(scrapy.Item):
id = scrapy.Field()
name = scrapy.Field()
description = scrapy.Field()
@dataclass
class TestItem:
id: str | None = None
name: str | None = None
description: str | None = None
.. currentmodule:: scrapy.spiders
@ -556,7 +557,6 @@ Let's now take a look at an example CrawlSpider with rules:
.. code-block:: python
import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
@ -576,7 +576,7 @@ Let's now take a look at an example CrawlSpider with rules:
def parse_item(self, response):
self.logger.info("Hi, this is an item page! %s", response.url)
item = scrapy.Item()
item = {}
item["id"] = response.xpath('//td[@id="item_id"]/text()').re(r"ID: (\d+)")
item["name"] = response.xpath('//td[@id="item_name"]/text()').get()
item["description"] = response.xpath(
@ -714,9 +714,9 @@ These spiders are pretty easy to use, let's have a look at one example:
)
item = TestItem()
item["id"] = node.xpath("@id").get()
item["name"] = node.xpath("name").get()
item["description"] = node.xpath("description").get()
item.id = node.xpath("@id").get()
item.name = node.xpath("name").get()
item.description = node.xpath("description").get()
return item
Basically what we did up there was to create a spider that downloads a feed from
@ -778,9 +778,9 @@ Let's see an example similar to the previous one, but using a
self.logger.info("Hi, this is a row!: %r", row)
item = TestItem()
item["id"] = row["id"]
item["name"] = row["name"]
item["description"] = row["description"]
item.id = row["id"]
item.name = row["name"]
item.description = row["description"]
return item

7
pyproject.toml
View File

@ -86,7 +86,6 @@ pattern = "^(?P<version>.+)$"
[tool.mypy]
strict = true
allow_any_generics = true # 67 errors
extra_checks = false # weird addErrback() errors
untyped_calls_exclude = [
"twisted",
@ -138,7 +137,7 @@ module = [
ignore_missing_imports = true
[tool.bumpversion]
current_version = "2.15.2"
current_version = "2.16.0"
commit = true
tag = true
tag_name = "{new_version}"
@ -158,6 +157,8 @@ parse = """(?P<major>0|[1-9]\\d*)\\.(?P<minor>0|[1-9]\\d*)"""
serialize = ["{major}.{minor}"]
[tool.coverage.run]
# sysmon, default on 3.14, is too slow: https://github.com/coveragepy/coveragepy/issues/2172
core = "ctrace"
branch = true
include = ["scrapy/*"]
omit = ["tests/*"]
@ -226,6 +227,7 @@ disable = [
"too-many-positional-arguments",
"too-many-public-methods",
"too-many-return-statements",
"undefined-variable",
"unused-argument",
"unused-variable",
"useless-import-alias", # used as a hint to mypy
@ -267,6 +269,7 @@ markers = [
"requires_botocore: marks tests that need botocore (but not boto3)",
"requires_boto3: marks tests that need botocore and boto3",
"requires_mitmproxy: marks tests that need mitmproxy",
"requires_internet: marks tests that need real Internet access",
]
filterwarnings = [
"ignore::DeprecationWarning:twisted.web.static",

2
scrapy/VERSION
View File

@ -1 +1 @@
2.15.2
2.16.0

2
scrapy/commands/genspider.py
View File

@ -147,7 +147,7 @@ class Command(ScrapyCommand):
name: str,
url: str,
template_name: str,
template_file: str | os.PathLike,
template_file: str | os.PathLike[str],
) -> None:
"""Generate the spider module, based on the given template"""
assert self.settings is not None

12
scrapy/contracts/__init__.py
View File

@ -27,7 +27,7 @@ class Contract:
request_cls: type[Request] | None = None
name: str
def __init__(self, method: Callable, *args: Any):
def __init__(self, method: Callable[..., Any], *args: Any):
self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook")
self.testcase_post = _create_testcase(method, f"@{self.name} post-hook")
self.args: tuple[Any, ...] = args
@ -105,7 +105,7 @@ class ContractsManager:
return methods
def extract_contracts(self, method: Callable) -> list[Contract]:
def extract_contracts(self, method: Callable[..., Any]) -> list[Contract]:
contracts: list[Contract] = []
assert method.__doc__ is not None
for line_ in method.__doc__.split("\n"):
@ -134,7 +134,9 @@ class ContractsManager:
return requests
def from_method(self, method: Callable, results: TestResult) -> Request | None:
def from_method(
self, method: Callable[..., Any], results: TestResult
) -> Request | None:
contracts = self.extract_contracts(method)
if contracts:
request_cls = Request
@ -170,7 +172,7 @@ class ContractsManager:
return None
def _clean_req(
self, request: Request, method: Callable, results: TestResult
self, request: Request, method: Callable[..., Any], results: TestResult
) -> None:
"""stop the request from returning objects and records any errors"""
@ -195,7 +197,7 @@ class ContractsManager:
request.errback = eb_wrapper
def _create_testcase(method: Callable, desc: str) -> TestCase:
def _create_testcase(method: Callable[..., Any], desc: str) -> TestCase:
spider = method.__self__.name # type: ignore[attr-defined]
class ContractTestCase(TestCase):

108
scrapy/core/downloader/contextfactory.py
View File

@ -1,13 +1,13 @@
from __future__ import annotations
import warnings
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, cast
from OpenSSL import SSL
from twisted.internet.ssl import (
AcceptableCiphers,
CertificateOptions,
TLSVersion,
optionsForClientTLS,
)
from twisted.web.client import BrowserLikePolicyForHTTPS
@ -16,19 +16,19 @@ from zope.interface.declarations import implementer
from zope.interface.verify import verifyObject
from scrapy.core.downloader.tls import (
_TWISTED_VERSION_MAP,
DEFAULT_CIPHERS,
_openssl_methods,
_ScrapyClientTLSOptions,
_ScrapyClientTLSOptions26,
openssl_methods,
)
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils._deps_compat import TWISTED_TLS_NEW_IMPL
from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.ssl import _get_cert_options_version_kwargs, _get_tls_version_limits
if TYPE_CHECKING:
from collections.abc import Generator
from twisted.internet._sslverify import ClientTLSOptions
# typing.Self requires Python 3.11
@ -38,24 +38,14 @@ if TYPE_CHECKING:
from scrapy.settings import BaseSettings
@contextmanager
def _filter_method_warning() -> Generator[None]:
with warnings.catch_warnings():
# Twisted deprecation, https://github.com/scrapy/scrapy/issues/3288
warnings.filterwarnings(
"ignore",
message=r"Passing method to twisted\.internet\.ssl\.CertificateOptions",
category=DeprecationWarning,
)
yield
@implementer(IPolicyForHTTPS)
class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
"""Non-peer-certificate verifying HTTPS context factory.
Default OpenSSL method is ``TLS_METHOD`` (also called ``SSLv23_METHOD``)
which allows TLS protocol negotiation.
Uses :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`,
:setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION`
to configure the :class:`~twisted.internet.ssl.CertificateOptions`
instance.
The purpose of this custom class is to provide a ``creatorForNetloc()``
method that returns a ``_ScrapyClientTLSOptions`` instance configured based
@ -64,15 +54,19 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
def __init__(
self,
method: int = SSL.SSLv23_METHOD, # noqa: S503
method: int | None = SSL.SSLv23_METHOD, # noqa: S503
tls_verbose_logging: bool = False,
tls_ciphers: str | None = None,
*args: Any,
verify_certificates: bool = False,
tls_min_version: TLSVersion | None = None,
tls_max_version: TLSVersion | None = None,
**kwargs: Any,
):
super().__init__(*args, **kwargs) # type: ignore[no-untyped-call]
self._ssl_method: int = method
self._ssl_method: int | None = method
self.tls_min_version: TLSVersion | None = tls_min_version
self.tls_max_version: TLSVersion | None = tls_max_version
self.tls_verbose_logging: bool = tls_verbose_logging # unused
self.tls_ciphers: AcceptableCiphers
if tls_ciphers:
@ -85,7 +79,7 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
def from_crawler(
cls,
crawler: Crawler,
method: int = SSL.SSLv23_METHOD, # noqa: S503
method: int | None = SSL.SSLv23_METHOD, # noqa: S503
*args: Any,
**kwargs: Any,
) -> Self:
@ -93,12 +87,21 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
)
tls_ciphers: str | None = crawler.settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
# DOWNLOADER_CLIENT_TLS_METHOD reading and handling should be also moved here
# when the deprecated load_context_factory_from_settings() is removed
tls_min_ver, tls_max_ver = _get_tls_version_limits(
crawler.settings, _TWISTED_VERSION_MAP.__getitem__
)
if tls_min_ver or tls_max_ver:
method = None
verify_certificates = crawler.settings.getbool("DOWNLOAD_VERIFY_CERTIFICATES")
return cls( # type: ignore[misc]
*args,
method=method,
tls_verbose_logging=tls_verbose_logging,
tls_ciphers=tls_ciphers,
tls_min_version=tls_min_ver,
tls_max_version=tls_max_ver,
verify_certificates=verify_certificates,
**kwargs,
)
@ -108,12 +111,23 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
return self._get_cert_options()
def _get_cert_options(self) -> CertificateOptions:
with _filter_method_warning():
return _ScrapyCertificateOptions(
method=self._ssl_method,
fixBrokenPeers=True,
acceptableCiphers=self.tls_ciphers,
return _ScrapyCertificateOptions(**self._get_cert_options_kwargs())
def _get_cert_options_kwargs(self) -> dict[str, Any]:
kwargs: dict[str, Any] = {
"fixBrokenPeers": True,
"acceptableCiphers": self.tls_ciphers,
}
if self.tls_min_version or self.tls_max_version:
kwargs.update(
_get_cert_options_version_kwargs(
self.tls_min_version, self.tls_max_version
)
)
# when ScrapyClientContextFactory is removed self._ssl_method can just be None by default
elif self._ssl_method != SSL.SSLv23_METHOD:
kwargs["method"] = self._ssl_method
return kwargs
# should be removed together with ScrapyClientContextFactory
def getContext(
@ -131,22 +145,16 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
return _ScrapyClientTLSOptions26(
self._get_cert_options()._makeTLSConnection,
hostname.decode("ascii"),
verbose_logging=self.tls_verbose_logging,
)
return _ScrapyClientTLSOptions(
hostname.decode("ascii"), # type: ignore[arg-type]
self._get_context(), # type: ignore[arg-type]
)
# Otherwise use the normal Twisted function.
# Note that this doesn't use self._get_context().
with _filter_method_warning():
return optionsForClientTLS( # type: ignore[no-any-return]
hostname=hostname.decode("ascii"),
extraCertificateOptions={
"method": self._ssl_method,
"acceptableCiphers": self.tls_ciphers,
},
)
return optionsForClientTLS( # type: ignore[no-any-return]
hostname=hostname.decode("ascii"),
extraCertificateOptions=self._get_cert_options_kwargs(),
)
ScrapyClientContextFactory = create_deprecated_class(
@ -171,12 +179,6 @@ class BrowserLikeContextFactory(_ScrapyClientContextFactory):
:meth:`creatorForNetloc` is the same as
:class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context
factory allows setting the TLS/SSL method to use.
The default OpenSSL method is ``TLS_METHOD`` (also called
``SSLv23_METHOD``) which allows TLS protocol negotiation.
As this overrides the parent ``creatorForNetloc()`` method, only
``self._ssl_method`` is used from the parent class.
"""
def __init__(self, *args: Any, **kwargs: Any):
@ -190,11 +192,10 @@ class BrowserLikeContextFactory(_ScrapyClientContextFactory):
super().__init__(*args, **kwargs)
def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
with _filter_method_warning():
return optionsForClientTLS( # type: ignore[no-any-return]
hostname=hostname.decode("ascii"),
extraCertificateOptions={"method": self._ssl_method},
)
return optionsForClientTLS( # type: ignore[no-any-return]
hostname=hostname.decode("ascii"),
extraCertificateOptions=self._get_cert_options_kwargs(),
)
@implementer(IPolicyForHTTPS)
@ -269,6 +270,16 @@ def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS:
Also passes values of other relevant settings to the factory class.
"""
tls_method_setting: str = crawler.settings["DOWNLOADER_CLIENT_TLS_METHOD"]
if tls_method_setting != "TLS":
warnings.warn(
"Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is"
" deprecated, please use DOWNLOAD_TLS_MIN_VERSION and/or"
" DOWNLOAD_TLS_MAX_VERSION instead.",
ScrapyDeprecationWarning,
stacklevel=2,
)
tls_method = _openssl_methods[tls_method_setting]
if crawler.settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] == "SENTINEL":
context_factory_cls = _ScrapyClientContextFactory
else: # pragma: no cover
@ -280,13 +291,12 @@ def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS:
context_factory_cls = load_object(
crawler.settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]
)
ssl_method = openssl_methods[crawler.settings.get("DOWNLOADER_CLIENT_TLS_METHOD")]
return cast(
"IPolicyForHTTPS",
build_from_crawler(
context_factory_cls,
crawler,
method=ssl_method,
method=tls_method,
),
)

25
scrapy/core/downloader/handlers/_base_http.py Normal file
View File

@ -0,0 +1,25 @@
from __future__ import annotations
from abc import ABC
from typing import TYPE_CHECKING
from .base import BaseDownloadHandler
if TYPE_CHECKING:
from scrapy.crawler import Crawler
class BaseHttpDownloadHandler(BaseDownloadHandler, ABC):
"""Base class for built-in HTTP download handlers."""
def __init__(self, crawler: Crawler):
super().__init__(crawler)
self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE")
self._default_warnsize: int = crawler.settings.getint("DOWNLOAD_WARNSIZE")
self._fail_on_dataloss: bool = crawler.settings.getbool(
"DOWNLOAD_FAIL_ON_DATALOSS"
)
self._tls_verbose_logging: bool = crawler.settings.getbool(
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
)
self._fail_on_dataloss_warned: bool = False

307
scrapy/core/downloader/handlers/_base_streaming.py Normal file
View File

@ -0,0 +1,307 @@
from __future__ import annotations
import base64
import logging
import time
from abc import ABC, abstractmethod
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Generic, NoReturn, TypedDict, TypeVar
from urllib.parse import quote, urlsplit
from scrapy import Request, signals
from scrapy.exceptions import (
DownloadCancelledError,
NotConfigured,
ResponseDataLossError,
)
from scrapy.utils._download_handlers import (
check_stop_download,
get_dataloss_msg,
get_maxsize_msg,
get_warnsize_msg,
make_response,
normalize_bind_address,
)
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.url import add_http_if_no_scheme
from ._base_http import BaseHttpDownloadHandler
if TYPE_CHECKING:
from collections.abc import AsyncIterable
from contextlib import AbstractAsyncContextManager
from ipaddress import IPv4Address, IPv6Address
from _typeshed import SizedBuffer
# typing.NotRequired requires Python 3.11
from typing_extensions import NotRequired
from scrapy.crawler import Crawler
from scrapy.http import Headers, Response
logger = logging.getLogger(__name__)
_ResponseT = TypeVar("_ResponseT")
class _BaseResponseArgs(TypedDict):
status: int
url: str
headers: Headers
certificate: NotRequired[Any]
ip_address: NotRequired[IPv4Address | IPv6Address | None]
protocol: str | None
class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_ResponseT]):
"""A base class for HTTP download handlers that follow the streaming logic flow."""
_DEFAULT_CONNECT_TIMEOUT: ClassVar[float] = 10
experimental: ClassVar[bool] = False
requires_asyncio: ClassVar[bool] = True
# require subclasses to disable proxies explicitly with an explanation
supports_proxies: ClassVar[bool] = True
supports_per_request_bindaddress: ClassVar[bool] = False
def __init__(self, crawler: Crawler):
if self.requires_asyncio and not is_asyncio_available(): # pragma: no cover
raise NotConfigured(
f"{type(self).__name__} requires the asyncio support. Make"
f" sure that you have either enabled the asyncio Twisted"
f" reactor in the TWISTED_REACTOR setting or disabled the"
f" TWISTED_REACTOR_ENABLED setting. See the asyncio documentation"
f" of Scrapy for more information."
)
self._check_deps_installed()
super().__init__(crawler)
if self.experimental:
logger.warning(
f"{type(self).__name__} is experimental and is not recommended for production use."
)
self._bind_address = normalize_bind_address(
crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
)
self._proxy_auth_encoding: str = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")
# these are useful for many handlers but used in different ways by them
self._pool_size_total: int = crawler.settings.getint("CONCURRENT_REQUESTS")
self._pool_size_per_host: int = crawler.settings.getint(
"CONCURRENT_REQUESTS_PER_DOMAIN"
)
@staticmethod
@abstractmethod
def _check_deps_installed() -> None:
"""Raise NotConfigured if the required deps are not installed."""
raise NotImplementedError
@abstractmethod
def _make_request(
self, request: Request, timeout: float
) -> AbstractAsyncContextManager[_ResponseT]:
"""Return an async context manager yielding the library-specific response.
Exceptions raised by the library should be reraised as Scrapy-specific ones.
"""
raise NotImplementedError
@staticmethod
@abstractmethod
def _extract_headers(response: _ResponseT) -> Headers:
"""Convert library-specific response headers to a
:class:`~scrapy.http.headers.Headers` object."""
raise NotImplementedError
@staticmethod
@abstractmethod
def _build_base_response_args(
response: _ResponseT, request: Request, headers: Headers
) -> _BaseResponseArgs:
"""Build kwargs for :func:`scrapy.utils._download_handlers.make_response`."""
raise NotImplementedError
@staticmethod
@abstractmethod
def _iter_body_chunks(response: _ResponseT) -> AsyncIterable[SizedBuffer]:
"""Return an async iterable yielding body chunks from the response."""
raise NotImplementedError
@staticmethod
@abstractmethod
def _is_dataloss_exception(exc: Exception) -> bool:
"""Return True if ``exc`` represents dataloss."""
raise NotImplementedError
def _log_tls_info(self, response: _ResponseT, request: Request) -> None:
"""Log TLS connection details, if possible."""
async def download_request(self, request: Request) -> Response:
if not self.supports_proxies and request.meta.get("proxy"):
raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.")
if not self.supports_per_request_bindaddress and request.meta.get(
"bindaddress"
):
logger.error(
f"The 'bindaddress' request meta key is not supported by"
f" {type(self).__name__} and will be ignored."
)
timeout: float = request.meta.get(
"download_timeout", self._DEFAULT_CONNECT_TIMEOUT
)
start_time = time.monotonic()
async with self._make_request(request, timeout) as response:
request.meta["download_latency"] = time.monotonic() - start_time
return await self._read_response(response, request)
async def _read_response(self, response: _ResponseT, request: Request) -> Response:
maxsize: int = request.meta.get("download_maxsize", self._default_maxsize)
warnsize: int = request.meta.get("download_warnsize", self._default_warnsize)
headers = self._extract_headers(response)
content_length = headers.get("Content-Length")
expected_size = int(content_length) if content_length is not None else None
if maxsize and expected_size and expected_size > maxsize:
self._cancel_maxsize(expected_size, maxsize, request, expected=True)
reached_warnsize = False
if warnsize and expected_size and expected_size > warnsize:
reached_warnsize = True
logger.warning(
get_warnsize_msg(expected_size, warnsize, request, expected=True)
)
make_response_base_args = self._build_base_response_args(
response, request, headers
)
if self._tls_verbose_logging:
self._log_tls_info(response, request)
if stop_download := check_stop_download(
signals.headers_received,
self.crawler,
request,
headers=headers,
body_length=expected_size,
):
return make_response(
**make_response_base_args,
stop_download=stop_download,
)
response_body = BytesIO()
bytes_received = 0
try:
async for chunk in self._iter_body_chunks(response):
response_body.write(chunk)
bytes_received += len(chunk)
if stop_download := check_stop_download(
signals.bytes_received, self.crawler, request, data=chunk
):
return make_response(
**make_response_base_args,
body=response_body.getvalue(),
stop_download=stop_download,
)
if maxsize and bytes_received > maxsize:
response_body.truncate(0)
self._cancel_maxsize(
bytes_received, maxsize, request, expected=False
)
if warnsize and bytes_received > warnsize and not reached_warnsize:
reached_warnsize = True
logger.warning(
get_warnsize_msg(
bytes_received, warnsize, request, expected=False
)
)
except Exception as e:
if not self._is_dataloss_exception(e):
raise
fail_on_dataloss: bool = request.meta.get(
"download_fail_on_dataloss", self._fail_on_dataloss
)
if not fail_on_dataloss:
return make_response(
**make_response_base_args,
body=response_body.getvalue(),
flags=["dataloss"],
)
if not self._fail_on_dataloss_warned:
logger.warning(get_dataloss_msg(request.url))
self._fail_on_dataloss_warned = True
raise ResponseDataLossError(str(e)) from e
return make_response(
**make_response_base_args,
body=response_body.getvalue(),
)
def _get_bind_address_host(self) -> str | None:
"""Return the host portion of the bind address.
Needed for handlers that don't support the bind port.
"""
if self._bind_address is None:
return None
host, port = self._bind_address
if port != 0:
logger.warning(
"DOWNLOAD_BIND_ADDRESS specifies a port (%s), but %s does not "
"support binding to a specific local port. Ignoring the port "
"and binding only to %r.",
port,
type(self).__name__,
host,
)
return host
@staticmethod
def _cancel_maxsize(
size: int, limit: int, request: Request, *, expected: bool
) -> NoReturn:
warning_msg = get_maxsize_msg(size, limit, request, expected=expected)
logger.warning(warning_msg)
raise DownloadCancelledError(warning_msg)
@staticmethod
def _extract_proxy(request: Request) -> tuple[str | None, str | None]:
"""Return a tuple of the proxy URL with a scheme and the value of the
Proxy-Authorization header.
This is useful for handlers that take the proxy headers separately.
"""
proxy: str | None = request.meta.get("proxy")
if not proxy:
return None, None
proxy = add_http_if_no_scheme(proxy)
auth_header: list[bytes] | None = request.headers.pop(
b"Proxy-Authorization", None
)
return proxy, auth_header[0].decode("ascii") if auth_header else None
def _extract_proxy_url_with_creds(self, request: Request) -> str | None:
"""Return the proxy URL with the userinfo added based on the
Proxy-Authorization header.
This is useful for handlers that cannot take the proxy headers
separately.
"""
proxy_url, auth_header = self._extract_proxy(request)
if proxy_url is None or auth_header is None:
return proxy_url
scheme, token = auth_header.split(" ", 1)
if scheme != "Basic":
raise ValueError(
f"Expected Basic auth in Proxy-Authorization, got {scheme}"
)
user, password = (
base64.b64decode(token).decode(self._proxy_auth_encoding).split(":", 1)
)
parts = urlsplit(proxy_url)
netloc = f"{quote(user)}:{quote(password)}@{parts.netloc}"
return parts._replace(netloc=netloc).geturl()

393
scrapy/core/downloader/handlers/_httpx.py
View File

@ -3,136 +3,165 @@
from __future__ import annotations
import ipaddress
import logging
import ssl
import time
from http.cookiejar import Cookie, CookieJar
from io import BytesIO
from typing import TYPE_CHECKING, Any, NoReturn, TypedDict
from contextlib import asynccontextmanager
from socket import gaierror
from typing import TYPE_CHECKING, ClassVar
from scrapy import Request, signals
from scrapy.exceptions import (
CannotResolveHostError,
DownloadCancelledError,
DownloadConnectionRefusedError,
DownloadFailedError,
DownloadTimeoutError,
NotConfigured,
ResponseDataLossError,
UnsupportedURLSchemeError,
)
from scrapy.http import Headers, Response
from scrapy.utils._download_handlers import (
BaseHttpDownloadHandler,
check_stop_download,
get_dataloss_msg,
get_maxsize_msg,
get_warnsize_msg,
make_response,
normalize_bind_address,
from scrapy.http import Headers
from scrapy.utils._download_handlers import NullCookieJar
from scrapy.utils.python import _iter_exc_causes
from scrapy.utils.ssl import (
_log_sslobj_debug_info,
_make_insecure_ssl_ctx,
_make_ssl_context,
)
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.ssl import _log_sslobj_debug_info, _make_ssl_context
from ._base_streaming import BaseStreamingDownloadHandler, _BaseResponseArgs
if TYPE_CHECKING:
from contextlib import AbstractAsyncContextManager
from http.client import HTTPResponse
from ipaddress import IPv4Address, IPv6Address
from urllib.request import Request as ULRequest
from collections.abc import AsyncIterator
from httpcore import AsyncNetworkStream
from scrapy import Request
from scrapy.crawler import Crawler
HAS_SOCKS = HAS_HTTP2 = False
try:
import httpx
except ImportError:
httpx = None # type: ignore[assignment]
else:
# a small hack to avoid importing these optional extras unconditionally
logger = logging.getLogger(__name__)
DOWNLOAD_FAILED_EXCEPTIONS: tuple[type[BaseException], ...] = (
httpx.RequestError,
httpx.InvalidURL,
)
try:
import h2.exceptions
class _BaseResponseArgs(TypedDict):
status: int
url: str
headers: Headers
ip_address: IPv4Address | IPv6Address
protocol: str
# workaround for (and from) https://github.com/encode/httpx/issues/2992
class _NullCookieJar(CookieJar): # pragma: no cover
"""A CookieJar that rejects all cookies."""
def extract_cookies(self, response: HTTPResponse, request: ULRequest) -> None:
HAS_HTTP2 = True
DOWNLOAD_FAILED_EXCEPTIONS += (h2.exceptions.InvalidBodyLengthError,)
except ImportError: # pragma: no cover
pass
def set_cookie(self, cookie: Cookie) -> None:
try:
import socksio.exceptions
HAS_SOCKS = True
DOWNLOAD_FAILED_EXCEPTIONS += (socksio.exceptions.ProtocolError,)
except ImportError: # pragma: no cover
pass
class HttpxDownloadHandler(BaseHttpDownloadHandler):
_DEFAULT_CONNECT_TIMEOUT = 10
if TYPE_CHECKING:
_Base = BaseStreamingDownloadHandler[httpx.Response]
else:
_Base = BaseStreamingDownloadHandler
class HttpxDownloadHandler(_Base):
experimental: ClassVar[bool] = True
def __init__(self, crawler: Crawler):
# we skip HttpxDownloadHandler tests with the non-asyncio reactor
if not is_asyncio_available(): # pragma: no cover
super().__init__(crawler)
self._verify_certificates: bool = crawler.settings.getbool(
"DOWNLOAD_VERIFY_CERTIFICATES"
)
self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED")
if self._enable_h2 and not HAS_HTTP2: # pragma: no cover
raise NotConfigured(
f"{type(self).__name__} requires the asyncio support. Make"
f" sure that you have either enabled the asyncio Twisted"
f" reactor in the TWISTED_REACTOR setting or disabled the"
f" TWISTED_REACTOR_ENABLED setting. See the asyncio"
f" documentation of Scrapy for more information."
f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed."
)
self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
self._bind_host: str | None = self._get_bind_address_host()
self._limits: httpx.Limits = httpx.Limits(
# hard limit on simultaneous connections
max_connections=self._pool_size_total,
# total number of idle connections in the pool (extra ones are closed)
max_keepalive_connections=self._pool_size_total,
)
self._default_client: httpx.AsyncClient = self._make_client()
# httpx doesn't support per-request proxies: https://github.com/encode/httpx/discussions/3183,
# so we keep a pool of clients per proxy URL. LRU eviction can be added here if needed.
self._proxy_clients: dict[str, httpx.AsyncClient] = {}
@staticmethod
def _check_deps_installed() -> None:
if httpx is None: # pragma: no cover
raise NotConfigured(
f"{type(self).__name__} requires the httpx library to be installed."
"HttpxDownloadHandler requires the httpx library to be installed."
)
super().__init__(crawler)
logger.warning(
"HttpxDownloadHandler is experimental and is not recommended for production use."
)
bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
bind_address = normalize_bind_address(bind_address)
self._bind_address: str | None = None
def _make_client(self, proxy_url: str | None = None) -> httpx.AsyncClient:
if proxy_url:
if proxy_url.startswith("https:") and not self._verify_certificates:
proxy_ssl_context = _make_insecure_ssl_ctx()
else:
proxy_ssl_context = None
proxy = httpx.Proxy(proxy_url, ssl_context=proxy_ssl_context)
else:
proxy = None
if bind_address is not None:
host, port = bind_address
if port != 0:
logger.warning(
"DOWNLOAD_BIND_ADDRESS specifies a port (%s), but %s does not "
"support binding to a specific local port. Ignoring the port "
"and binding only to %r.",
port,
type(self).__name__,
host,
)
self._bind_address = host
self._client = httpx.AsyncClient(
cookies=_NullCookieJar(),
client = httpx.AsyncClient(
cookies=NullCookieJar(),
transport=httpx.AsyncHTTPTransport(
verify=_make_ssl_context(crawler.settings),
local_address=self._bind_address,
verify=self._ssl_context,
local_address=self._bind_host,
http2=self._enable_h2,
limits=self._limits,
trust_env=False,
proxy=proxy,
),
)
# https://github.com/encode/httpx/discussions/1566
for header_name in ("accept", "accept-encoding", "user-agent"):
self._client.headers.pop(header_name, None)
client.headers.pop(header_name, None)
return client
async def download_request(self, request: Request) -> Response:
self._warn_unsupported_meta(request.meta)
def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient:
if proxy_url is None:
return self._default_client
if cached := self._proxy_clients.get(proxy_url):
return cached
client = self._make_client(proxy_url)
self._proxy_clients[proxy_url] = client
return client
@asynccontextmanager
async def _make_request(
self, request: Request, timeout: float
) -> AsyncIterator[httpx.Response]:
proxy = self._extract_proxy_url_with_creds(request)
if proxy and proxy.startswith("socks") and not HAS_SOCKS: # pragma: no cover
raise ValueError(
f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed."
)
client = self._get_client(proxy)
timeout: float = request.meta.get(
"download_timeout", self._DEFAULT_CONNECT_TIMEOUT
)
start_time = time.monotonic()
try:
async with self._get_httpx_response(request, timeout) as httpx_response:
request.meta["download_latency"] = time.monotonic() - start_time
return await self._read_response(httpx_response, request)
async with client.stream(
request.method,
request.url,
content=request.body,
headers=request.headers.to_tuple_list(),
timeout=timeout,
) as response:
yield response
except httpx.TimeoutException as e:
raise DownloadTimeoutError(
f"Getting {request.url} took longer than {timeout} seconds."
@ -140,168 +169,58 @@ class HttpxDownloadHandler(BaseHttpDownloadHandler):
except httpx.UnsupportedProtocol as e:
raise UnsupportedURLSchemeError(str(e)) from e
except httpx.ConnectError as e:
error_message = str(e)
if (
"Name or service not known" in error_message
or "getaddrinfo failed" in error_message
or "nodename nor servname" in error_message
or "Temporary failure in name resolution" in error_message
):
raise CannotResolveHostError(error_message) from e
if any(isinstance(c, gaierror) for c in _iter_exc_causes(e)):
raise CannotResolveHostError(str(e)) from e
raise DownloadConnectionRefusedError(str(e)) from e
except httpx.NetworkError as e:
except httpx.ProxyError as e:
raise DownloadConnectionRefusedError(str(e)) from e
except DOWNLOAD_FAILED_EXCEPTIONS as e:
raise DownloadFailedError(str(e)) from e
except httpx.RemoteProtocolError as e:
raise DownloadFailedError(str(e)) from e
def _warn_unsupported_meta(self, meta: dict[str, Any]) -> None:
if meta.get("bindaddress"):
# configurable only per-client:
# https://github.com/encode/httpx/issues/755#issuecomment-2746121794
logger.error(
f"The 'bindaddress' request meta key is not supported by"
f" {type(self).__name__} and will be ignored."
)
if meta.get("proxy"):
# configurable only per-client:
# https://github.com/encode/httpx/issues/486
logger.error(
f"The 'proxy' request meta key is not supported by"
f" {type(self).__name__} and will be ignored."
)
def _get_httpx_response(
self, request: Request, timeout: float
) -> AbstractAsyncContextManager[httpx.Response]:
return self._client.stream(
request.method,
request.url,
content=request.body,
headers=request.headers.to_tuple_list(),
timeout=timeout,
)
async def _read_response(
self, httpx_response: httpx.Response, request: Request
) -> Response:
maxsize: int = request.meta.get("download_maxsize", self._default_maxsize)
warnsize: int = request.meta.get("download_warnsize", self._default_warnsize)
content_length = httpx_response.headers.get("Content-Length")
expected_size = int(content_length) if content_length is not None else None
if maxsize and expected_size and expected_size > maxsize:
self._cancel_maxsize(expected_size, maxsize, request, expected=True)
reached_warnsize = False
if warnsize and expected_size and expected_size > warnsize:
reached_warnsize = True
logger.warning(
get_warnsize_msg(expected_size, warnsize, request, expected=True)
)
headers = Headers(httpx_response.headers.multi_items())
network_stream: AsyncNetworkStream = httpx_response.extensions["network_stream"]
make_response_base_args: _BaseResponseArgs = {
"status": httpx_response.status_code,
"url": request.url,
"headers": headers,
"ip_address": self._get_server_ip(network_stream),
"protocol": httpx_response.http_version,
}
self._log_tls_info(network_stream)
if stop_download := check_stop_download(
signals.headers_received,
self.crawler,
request,
headers=headers,
body_length=expected_size,
):
return make_response(
**make_response_base_args,
stop_download=stop_download,
)
response_body = BytesIO()
bytes_received = 0
try:
async for chunk in httpx_response.aiter_raw():
response_body.write(chunk)
bytes_received += len(chunk)
if stop_download := check_stop_download(
signals.bytes_received, self.crawler, request, data=chunk
):
return make_response(
**make_response_base_args,
body=response_body.getvalue(),
stop_download=stop_download,
)
if maxsize and bytes_received > maxsize:
response_body.truncate(0)
self._cancel_maxsize(
bytes_received, maxsize, request, expected=False
)
if warnsize and bytes_received > warnsize and not reached_warnsize:
reached_warnsize = True
logger.warning(
get_warnsize_msg(
bytes_received, warnsize, request, expected=False
)
)
except httpx.RemoteProtocolError as e:
# special handling of the dataloss case
if (
"peer closed connection without sending complete message body"
not in str(e)
):
raise
fail_on_dataloss: bool = request.meta.get(
"download_fail_on_dataloss", self._fail_on_dataloss
)
if not fail_on_dataloss:
return make_response(
**make_response_base_args,
body=response_body.getvalue(),
flags=["dataloss"],
)
self._log_dataloss_warning(request.url)
raise ResponseDataLossError(str(e)) from e
return make_response(
**make_response_base_args,
body=response_body.getvalue(),
)
@staticmethod
def _get_server_ip(network_stream: AsyncNetworkStream) -> IPv4Address | IPv6Address:
extra_server_addr = network_stream.get_extra_info("server_addr")
return ipaddress.ip_address(extra_server_addr[0])
def _extract_headers(response: httpx.Response) -> Headers:
return Headers(response.headers.multi_items())
def _log_tls_info(self, network_stream: AsyncNetworkStream) -> None:
if not self._tls_verbose_logging:
return
@staticmethod
def _build_base_response_args(
response: httpx.Response,
request: Request,
headers: Headers,
) -> _BaseResponseArgs:
network_stream: AsyncNetworkStream = response.extensions["network_stream"]
server_addr = network_stream.get_extra_info("server_addr")
ip_address = ipaddress.ip_address(server_addr[0])
ssl_object = network_stream.get_extra_info("ssl_object")
if isinstance(ssl_object, ssl.SSLObject):
cert = ssl_object.getpeercert(binary_form=True)
else:
cert = None
return {
"status": response.status_code,
"url": request.url,
"headers": headers,
"certificate": cert,
"ip_address": ip_address,
"protocol": response.http_version,
}
@staticmethod
def _iter_body_chunks(response: httpx.Response) -> AsyncIterator[bytes]:
return response.aiter_raw()
@staticmethod
def _is_dataloss_exception(exc: Exception) -> bool:
return isinstance(
exc, httpx.RemoteProtocolError
) and "peer closed connection without sending complete message body" in str(exc)
def _log_tls_info(self, response: httpx.Response, request: Request) -> None:
network_stream: AsyncNetworkStream = response.extensions["network_stream"]
extra_ssl_object = network_stream.get_extra_info("ssl_object")
if isinstance(extra_ssl_object, ssl.SSLObject):
_log_sslobj_debug_info(extra_ssl_object)
def _log_dataloss_warning(self, url: str) -> None:
if self._fail_on_dataloss_warned:
return
logger.warning(get_dataloss_msg(url))
self._fail_on_dataloss_warned = True
@staticmethod
def _cancel_maxsize(
size: int, limit: int, request: Request, *, expected: bool
) -> NoReturn:
warning_msg = get_maxsize_msg(size, limit, request, expected=expected)
logger.warning(warning_msg)
raise DownloadCancelledError(warning_msg)
async def close(self) -> None:
await self._client.aclose()
await self._default_client.aclose()
for client in self._proxy_clients.values():
await client.aclose()

36
scrapy/core/downloader/handlers/http11.py
View File

@ -41,7 +41,6 @@ from scrapy.exceptions import (
)
from scrapy.http import Headers, Response
from scrapy.utils._download_handlers import (
BaseHttpDownloadHandler,
check_stop_download,
get_dataloss_msg,
get_maxsize_msg,
@ -57,6 +56,8 @@ from scrapy.utils.python import to_bytes, to_unicode
from scrapy.utils.ssl import _log_ssl_conn_debug_info
from scrapy.utils.url import add_http_if_no_scheme
from ._base_http import BaseHttpDownloadHandler
if TYPE_CHECKING:
from twisted.internet.base import ReactorBase
from twisted.internet.interfaces import IConsumer
@ -111,7 +112,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
"download_warnsize", "DOWNLOAD_WARNSIZE"
)
agent = ScrapyAgent(
agent = _ScrapyAgent(
contextFactory=self._contextFactory,
bindAddress=self._bind_address,
pool=self._pool,
@ -161,7 +162,7 @@ class TunnelError(Exception):
"""An HTTP CONNECT tunnel could not be established by the proxy."""
class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
"""An endpoint that tunnels through proxies to allow HTTPS downloads. To
accomplish that, this endpoint sends an HTTP CONNECT to the proxy.
The HTTP CONNECT is always sent when using this endpoint, I think this could
@ -197,7 +198,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def requestTunnel(self, protocol: Protocol) -> Protocol:
"""Asks the proxy to open a tunnel."""
assert protocol.transport
tunnelReq = tunnel_request_data(
tunnelReq = _tunnel_request_data(
self._tunneledHost, self._tunneledPort, self._proxyAuthHeader
)
protocol.transport.write(tunnelReq)
@ -221,7 +222,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
if b"\r\n\r\n" not in self._connectBuffer:
return
self._protocol.dataReceived = self._protocolDataReceived # type: ignore[method-assign]
respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
respm = _TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
if respm and int(respm.group("status")) == 200:
# set proper Server Name Indication extension
sslOptions = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc]
@ -258,18 +259,18 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
return self._tunnelReadyDeferred
def tunnel_request_data(
def _tunnel_request_data(
host: str, port: int, proxy_auth_header: bytes | None = None
) -> bytes:
r"""
Return binary content of a CONNECT request.
>>> from scrapy.utils.python import to_unicode as s
>>> s(tunnel_request_data("example.com", 8080))
>>> s(_tunnel_request_data("example.com", 8080))
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
>>> s(tunnel_request_data("example.com", 8080, b"123"))
>>> s(_tunnel_request_data("example.com", 8080, b"123"))
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n'
>>> s(tunnel_request_data(b"example.com", "8090"))
>>> s(_tunnel_request_data(b"example.com", "8090"))
'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'
"""
host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port))
@ -281,7 +282,7 @@ def tunnel_request_data(
return tunnel_req
class TunnelingAgent(Agent):
class _TunnelingAgent(Agent):
"""An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
downloads. It may look strange that we have chosen to subclass Agent and not
ProxyAgent but consider that after the tunnel is opened the proxy is
@ -303,8 +304,8 @@ class TunnelingAgent(Agent):
self._proxyConf: tuple[str, int, bytes | None] = proxyConf
self._contextFactory: IPolicyForHTTPS = contextFactory
def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint:
return TunnelingTCP4ClientEndpoint(
def _getEndpoint(self, uri: URI) -> _TunnelingTCP4ClientEndpoint:
return _TunnelingTCP4ClientEndpoint(
reactor=self._reactor,
host=uri.host,
port=uri.port,
@ -381,10 +382,7 @@ class _ScrapyProxyAgent(Agent):
)
class ScrapyAgent:
_Agent = Agent
_TunnelingAgent = TunnelingAgent
class _ScrapyAgent:
def __init__(
self,
*,
@ -430,7 +428,7 @@ class ScrapyAgent:
assert proxy_host is not None
proxyAuth = request.headers.get(b"Proxy-Authorization", None)
proxyConf = (proxy_host, proxy_port, proxyAuth)
return self._TunnelingAgent(
return _TunnelingAgent(
reactor=reactor,
proxyConf=proxyConf,
contextFactory=self._contextFactory,
@ -447,7 +445,7 @@ class ScrapyAgent:
pool=self._pool,
)
return self._Agent( # type: ignore[no-untyped-call]
return Agent(
reactor=reactor,
contextFactory=self._contextFactory,
connectTimeout=timeout,
@ -465,7 +463,7 @@ class ScrapyAgent:
url = urldefrag(request.url)[0]
method = to_bytes(request.method)
headers = TxHeaders(request.headers)
if isinstance(agent, self._TunnelingAgent):
if isinstance(agent, _TunnelingAgent):
headers.removeHeader(b"Proxy-Authorization")
bodyproducer = _RequestBodyProducer(request.body) if request.body else None
start_time = monotonic()

8
scrapy/core/downloader/handlers/http2.py
View File

@ -49,7 +49,7 @@ class H2DownloadHandler(BaseDownloadHandler):
raise UnsupportedURLSchemeError(
f"{type(self).__name__} doesn't support plain HTTP."
)
agent = ScrapyH2Agent(
agent = _ScrapyH2Agent(
context_factory=self._context_factory,
pool=self._pool,
bind_address=self._bind_address,
@ -65,9 +65,7 @@ class H2DownloadHandler(BaseDownloadHandler):
self._pool.close_connections()
class ScrapyH2Agent:
_Agent = H2Agent
class _ScrapyH2Agent:
def __init__(
self,
context_factory: IPolicyForHTTPS,
@ -89,7 +87,7 @@ class ScrapyH2Agent:
raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.")
bind_address = request.meta.get("bindaddress") or self._bind_address
bind_address = normalize_bind_address(bind_address)
return self._Agent(
return H2Agent(
reactor=reactor,
context_factory=self._context_factory,
connect_timeout=timeout,

6
scrapy/core/downloader/handlers/s3.py
View File

@ -1,6 +1,6 @@
from __future__ import annotations
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any, cast
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.exceptions import NotConfigured
@ -9,6 +9,8 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
from collections.abc import Mapping
from scrapy import Request
from scrapy.crawler import Crawler
from scrapy.http import Response
@ -59,7 +61,7 @@ class S3DownloadHandler(BaseDownloadHandler):
awsrequest = botocore.awsrequest.AWSRequest(
method=request.method,
url=f"{scheme}://s3.amazonaws.com/{bucket}{path}",
headers=request.headers.to_unicode_dict(),
headers=cast("Mapping[str, Any]", request.headers.to_unicode_dict()),
data=request.body,
)
assert self._signer

10
scrapy/core/downloader/middleware.py
View File

@ -8,7 +8,7 @@ from __future__ import annotations
import warnings
from functools import wraps
from typing import TYPE_CHECKING, Any, cast
from typing import TYPE_CHECKING, Any
from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
from scrapy.http import Request, Response
@ -87,7 +87,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
result = await self._process_exception(ex, request)
return await self._process_response(result, request)
def _handle_mw_method(self, method: Callable, **kwargs: Any) -> Any:
def _handle_mw_method(self, method: Callable[..., Any], **kwargs: Any) -> Any:
if method in self._mw_methods_requiring_spider:
kwargs["spider"] = self._spider
@ -99,7 +99,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
download_func: Callable[[Request], Coroutine[Any, Any, Response]],
) -> Response | Request:
for method in self.methods["process_request"]:
method = cast("Callable", method)
assert method is not None
response = await ensure_awaitable(
self._handle_mw_method(method, request=request),
_warn=global_object_name(method),
@ -122,7 +122,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
return response
for method in self.methods["process_response"]:
method = cast("Callable", method)
assert method is not None
response = await ensure_awaitable(
self._handle_mw_method(method, request=request, response=response),
_warn=global_object_name(method),
@ -141,7 +141,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
self, exception: Exception, request: Request | Response
) -> Response | Request:
for method in self.methods["process_exception"]:
method = cast("Callable", method)
assert method is not None
response = await ensure_awaitable(
self._handle_mw_method(method, request=request, exception=exception),
_warn=global_object_name(method),

61
scrapy/core/downloader/tls.py
View File

@ -1,6 +1,7 @@
from __future__ import annotations
import logging
import warnings
from typing import TYPE_CHECKING, Any
from OpenSSL import SSL
@ -18,8 +19,9 @@ from service_identity.pyopenssl import (
verify_ip_address,
)
from twisted.internet._sslverify import ClientTLSOptions
from twisted.internet.ssl import AcceptableCiphers
from twisted.internet.ssl import AcceptableCiphers, TLSVersion
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import create_deprecated_class
if TYPE_CHECKING:
@ -32,17 +34,37 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
METHOD_TLS = "TLS"
METHOD_TLSv10 = "TLSv1.0"
METHOD_TLSv11 = "TLSv1.1"
METHOD_TLSv12 = "TLSv1.2"
_openssl_methods: dict[str, int] = {
"TLS": SSL.SSLv23_METHOD, # protocol negotiation (recommended)
"TLSv1.0": SSL.TLSv1_METHOD, # TLS 1.0 only
"TLSv1.1": SSL.TLSv1_1_METHOD, # TLS 1.1 only
"TLSv1.2": SSL.TLSv1_2_METHOD, # TLS 1.2 only
}
openssl_methods: dict[str, int] = {
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only
METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only
METHOD_TLSv12: SSL.TLSv1_2_METHOD, # TLS 1.2 only
def __getattr__(name: str) -> Any:
deprecated = {
"METHOD_TLS": "TLS",
"METHOD_TLSv10": "TLSv1.0",
"METHOD_TLSv11": "TLSv1.1",
"METHOD_TLSv12": "TLSv1.2",
"openssl_methods": _openssl_methods,
}
if name in deprecated:
warnings.warn(
f"scrapy.core.downloader.tls.{name} is deprecated.",
ScrapyDeprecationWarning,
stacklevel=2,
)
return deprecated[name]
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
_TWISTED_VERSION_MAP: dict[str, TLSVersion] = {
"TLSv1.0": TLSVersion.TLSv1_0,
"TLSv1.1": TLSVersion.TLSv1_1,
"TLSv1.2": TLSVersion.TLSv1_2,
"TLSv1.3": TLSVersion.TLSv1_3,
}
@ -107,34 +129,23 @@ class _ScrapyClientTLSOptions26(ClientTLSOptions):
logging warnings.
Instances of this class are returned from
:class:`.ScrapyClientContextFactory`.
:class:`._ScrapyClientContextFactory`.
This class is used on Twisted 26.4.0 and newer.
"""
def __init__(
self,
createConnection: Callable[[TLSMemoryBIOProtocol], SSL.Connection],
hostname: str,
verbose_logging: bool = False,
):
super().__init__(createConnection, hostname)
self.verbose_logging: bool = verbose_logging
def clientConnectionForTLS(
self, tlsProtocol: TLSMemoryBIOProtocol
) -> SSL.Connection:
"""This method is needed to override the verify callback."""
conn = super().clientConnectionForTLS(tlsProtocol)
callback = self._verifyCB(
self._hostnameIsDnsName, self._hostnameASCII, self.verbose_logging
)
callback = self._verifyCB(self._hostnameIsDnsName, self._hostnameASCII)
conn.set_verify(SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, callback)
return conn
@staticmethod
def _verifyCB(
hostIsDNS: bool, hostnameASCII: str, verbose_logging: bool
hostIsDNS: bool, hostnameASCII: str
) -> Callable[[SSL.Connection, X509, int, int, int], bool]:
svcid: ServiceID = (
DNS_ID(hostnameASCII) if hostIsDNS else IPAddress_ID(hostnameASCII)
@ -145,7 +156,7 @@ class _ScrapyClientTLSOptions26(ClientTLSOptions):
) -> bool:
if depth != 0:
# We are only verifying the leaf certificate.
return bool(ok)
return True
try:
verify_service_identity(extract_patterns(cert), [svcid], [])

2
scrapy/core/scraper.py
View File

@ -125,7 +125,7 @@ class Scraper:
def _check_deprecated_itemproc_method(self, method: str) -> None:
itemproc_cls = type(self.itemproc)
if not hasattr(self.itemproc, "process_item_async"):
if not hasattr(self.itemproc, f"{method}_async"):
warnings.warn(
f"{global_object_name(itemproc_cls)} doesn't define a {method}_async() method,"
f" this is deprecated and the method will be required in future Scrapy versions.",

261
scrapy/core/spidermw.py
View File

@ -9,29 +9,28 @@ from __future__ import annotations
import logging
from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
from functools import wraps
from inspect import isasyncgenfunction, iscoroutine
from inspect import isasyncgenfunction
from itertools import islice
from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar
from warnings import warn
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.python.failure import Failure
from scrapy import Request, Spider
from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
from scrapy.http import Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
from scrapy.utils.asyncgen import as_async_generator
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import (
_defer_sleep_async,
deferred_from_coro,
maybe_deferred_to_future,
)
from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_name
from scrapy.utils.python import MutableAsyncChain, global_object_name
if TYPE_CHECKING:
from collections.abc import Generator
from twisted.internet.defer import Deferred
from scrapy.settings import BaseSettings
@ -46,10 +45,6 @@ ScrapeFunc: TypeAlias = Callable[
]
def _isiterable(o: Any) -> bool:
return isinstance(o, (Iterable, AsyncIterator))
class SpiderMiddlewareManager(MiddlewareManager):
component_name = "spider middleware"
@ -67,13 +62,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
if hasattr(mw, "process_start"):
self.methods["process_start"].appendleft(mw.process_start)
process_spider_output = self._get_async_method_pair(mw, "process_spider_output")
process_spider_output = self._get_process_spider_output(mw)
self.methods["process_spider_output"].appendleft(process_spider_output)
if callable(process_spider_output):
if process_spider_output is not None:
self._check_mw_method_spider_arg(process_spider_output)
elif isinstance(process_spider_output, tuple):
for m in process_spider_output:
self._check_mw_method_spider_arg(m)
process_spider_exception = getattr(mw, "process_spider_exception", None)
self.methods["process_spider_exception"].appendleft(process_spider_exception)
@ -87,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
request: Request,
) -> Iterable[_T] | AsyncIterator[_T]:
for method in self.methods["process_spider_input"]:
method = cast("Callable", method)
assert method is not None
try:
if method in self._mw_methods_requiring_spider:
result = method(response=response, spider=self._spider)
@ -105,48 +97,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
return await scrape_func(Failure(), request)
return await scrape_func(response, request)
def _evaluate_iterable(
self,
response: Response,
iterable: Iterable[_T] | AsyncIterator[_T],
exception_processor_index: int,
recover_to: MutableChain[_T] | MutableAsyncChain[_T],
) -> Iterable[_T] | AsyncIterator[_T]:
if isinstance(iterable, AsyncIterator):
return self._process_async(
response,
iterable,
exception_processor_index,
cast("MutableAsyncChain[_T]", recover_to),
)
return self._process_sync(
response,
iterable,
exception_processor_index,
cast("MutableChain[_T]", recover_to),
)
def _process_sync(
self,
response: Response,
iterable: Iterable[_T],
exception_processor_index: int,
recover_to: MutableChain[_T],
) -> Iterable[_T]:
try:
yield from iterable
except Exception as ex:
exception_result = cast(
"Failure | MutableChain[_T]",
self._process_spider_exception(response, ex, exception_processor_index),
)
if isinstance(exception_result, Failure):
raise
assert isinstance(recover_to, MutableChain)
recover_to.extend(exception_result)
async def _process_async(
async def _evaluate_iterable(
self,
response: Response,
iterable: AsyncIterator[_T],
@ -157,13 +108,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
async for r in iterable:
yield r
except Exception as ex:
exception_result = cast(
"Failure | MutableAsyncChain[_T]",
self._process_spider_exception(response, ex, exception_processor_index),
exception_result: MutableAsyncChain[_T] = self._process_spider_exception(
response, ex, exception_processor_index
)
if isinstance(exception_result, Failure):
raise
assert isinstance(recover_to, MutableAsyncChain)
recover_to.extend(exception_result)
def _process_spider_exception(
@ -171,7 +118,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
response: Response,
exception: Exception,
start_index: int = 0,
) -> MutableChain[_T] | MutableAsyncChain[_T]:
) -> MutableAsyncChain[_T]:
# don't handle _InvalidOutput exception
if isinstance(exception, _InvalidOutput):
raise exception
@ -181,28 +128,18 @@ class SpiderMiddlewareManager(MiddlewareManager):
for method_index, method in enumerate(method_list, start=start_index):
if method is None:
continue
method = cast("Callable", method)
if method in self._mw_methods_requiring_spider:
result = method(
response=response, exception=exception, spider=self._spider
)
else:
result = method(response=response, exception=exception)
if _isiterable(result):
if isinstance(result, (Iterable, AsyncIterator)):
# stop exception handling by handing control over to the
# process_spider_output chain if an iterable has been returned
dfd: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = (
self._process_spider_output(response, result, method_index + 1)
)
# _process_spider_output() returns a Deferred only because of downgrading so this can be
# simplified when downgrading is removed.
if dfd.called:
# the result is available immediately if _process_spider_output didn't do downgrading
return cast("MutableChain[_T] | MutableAsyncChain[_T]", dfd.result)
# we forbid waiting here because otherwise we would need to return a deferred from
# _process_spider_exception too, which complicates the architecture
msg = f"Async iterable returned from {global_object_name(method)} cannot be downgraded"
raise _InvalidOutput(msg)
if isinstance(result, Iterable):
result = as_async_generator(result)
return self._process_spider_output(response, result, method_index + 1)
if result is None:
continue
msg = (
@ -212,124 +149,35 @@ class SpiderMiddlewareManager(MiddlewareManager):
raise _InvalidOutput(msg)
raise exception
# This method cannot be made async def, as _process_spider_exception relies on the Deferred result
# being available immediately which doesn't work when it's a wrapped coroutine.
# It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed.
@inlineCallbacks
def _process_spider_output( # noqa: PLR0912
def _process_spider_output(
self,
response: Response,
result: Iterable[_T] | AsyncIterator[_T],
result: AsyncIterator[_T],
start_index: int = 0,
) -> Generator[Deferred[Any], Any, MutableChain[_T] | MutableAsyncChain[_T]]:
) -> MutableAsyncChain[_T]:
# items in this iterable do not need to go through the process_spider_output
# chain, they went through it already from the process_spider_exception method
recovered: MutableChain[_T] | MutableAsyncChain[_T]
last_result_is_async = isinstance(result, AsyncIterator)
recovered = MutableAsyncChain() if last_result_is_async else MutableChain()
# There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async.
# 1. def foo. Sync iterables are passed as is, async ones are downgraded.
# 2. async def foo. Sync iterables are upgraded, async ones are passed as is.
# 3. def foo + async def foo_async. Iterables are passed to the respective method.
# Storing methods and method tuples in the same list is weird but we should be able to roll this back
# when we drop this compatibility feature.
recovered: MutableAsyncChain[_T] = MutableAsyncChain()
method_list = islice(self.methods["process_spider_output"], start_index, None)
for method_index, method_pair in enumerate(method_list, start=start_index):
if method_pair is None:
for method_index, method in enumerate(method_list, start=start_index):
if method is None:
continue
need_upgrade = need_downgrade = False
if isinstance(method_pair, tuple):
# This tuple handling is only needed until _async compatibility methods are removed.
method_sync, method_async = method_pair
method = method_async if last_result_is_async else method_sync
if method in self._mw_methods_requiring_spider:
result = method(response=response, result=result, spider=self._spider)
else:
method = method_pair
if not last_result_is_async and isasyncgenfunction(method):
need_upgrade = True
elif last_result_is_async and not isasyncgenfunction(method):
need_downgrade = True
try:
if need_upgrade:
# Iterable -> AsyncIterator
result = as_async_generator(result)
elif need_downgrade:
logger.warning(
f"Async iterable passed to {global_object_name(method)} was"
f" downgraded to a non-async one. This is deprecated and will"
f" stop working in a future version of Scrapy. Please see"
f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users"
f" for more information."
)
assert isinstance(result, AsyncIterator)
# AsyncIterator -> Iterable
result = yield deferred_from_coro(collect_asyncgen(result))
if isinstance(recovered, AsyncIterator):
recovered_collected = yield deferred_from_coro(
collect_asyncgen(recovered)
)
recovered = MutableChain(recovered_collected)
# might fail directly if the output value is not a generator
if method in self._mw_methods_requiring_spider:
result = method(
response=response, result=result, spider=self._spider
)
else:
result = method(response=response, result=result)
except Exception as ex:
exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = (
self._process_spider_exception(response, ex, method_index + 1)
)
if isinstance(exception_result, Failure):
raise
return exception_result
if _isiterable(result):
result = self._evaluate_iterable(
response, result, method_index + 1, recovered
)
else:
if iscoroutine(result):
result.close() # Silence warning about not awaiting
msg = (
f"{global_object_name(method)} must be an asynchronous "
f"generator (i.e. use yield)"
)
else:
msg = (
f"{global_object_name(method)} must return an iterable, got "
f"{type(result)}"
)
raise _InvalidOutput(msg)
last_result_is_async = isinstance(result, AsyncIterator)
if last_result_is_async:
return MutableAsyncChain(result, recovered)
return MutableChain(result, recovered) # type: ignore[arg-type]
result = method(response=response, result=result)
result = self._evaluate_iterable(
response, result, method_index + 1, recovered
)
return MutableAsyncChain(result, recovered)
async def _process_callback_output(
self,
response: Response,
result: Iterable[_T] | AsyncIterator[_T],
) -> MutableChain[_T] | MutableAsyncChain[_T]:
recovered: MutableChain[_T] | MutableAsyncChain[_T]
if isinstance(result, AsyncIterator):
recovered = MutableAsyncChain()
else:
recovered = MutableChain()
self, response: Response, result: AsyncIterator[_T]
) -> MutableAsyncChain[_T]:
recovered: MutableAsyncChain[_T] = MutableAsyncChain()
result = self._evaluate_iterable(response, result, 0, recovered)
result = await maybe_deferred_to_future(
cast(
"Deferred[Iterable[_T] | AsyncIterator[_T]]",
self._process_spider_output(response, result),
)
)
if isinstance(result, AsyncIterator):
return MutableAsyncChain(result, recovered)
if isinstance(recovered, AsyncIterator):
recovered_collected = await collect_asyncgen(recovered)
recovered = MutableChain(recovered_collected)
return MutableChain(result, recovered)
result = self._process_spider_output(response, result)
return MutableAsyncChain(result, recovered)
def scrape_response(
self,
@ -340,7 +188,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
response: Response,
request: Request,
spider: Spider,
) -> Deferred[MutableChain[_T] | MutableAsyncChain[_T]]: # pragma: no cover
) -> Deferred[MutableAsyncChain[_T]]: # pragma: no cover
warn(
"SpiderMiddlewareManager.scrape_response() is deprecated, use scrape_response_async() instead",
ScrapyDeprecationWarning,
@ -363,7 +211,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
scrape_func: ScrapeFunc[_T],
response: Response,
request: Request,
) -> MutableChain[_T] | MutableAsyncChain[_T]:
) -> MutableAsyncChain[_T]:
if not self.crawler:
raise RuntimeError(
"scrape_response_async() called on a SpiderMiddlewareManager"
@ -373,7 +221,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
it: Iterable[_T] | AsyncIterator[_T] = await self._process_spider_input(
scrape_func, response, request
)
return await self._process_callback_output(response, it)
ait = it if isinstance(it, AsyncIterator) else as_async_generator(it)
return await self._process_callback_output(response, ait)
except Exception as ex:
await _defer_sleep_async()
return self._process_spider_exception(response, ex)
@ -399,27 +248,27 @@ class SpiderMiddlewareManager(MiddlewareManager):
# This method is only needed until _async compatibility methods are removed.
@staticmethod
def _get_async_method_pair(
mw: Any, methodname: str
) -> Callable | tuple[Callable, Callable] | None:
normal_method: Callable | None = getattr(mw, methodname, None)
methodname_async = methodname + "_async"
async_method: Callable | None = getattr(mw, methodname_async, None)
def _get_process_spider_output(mw: Any) -> Callable[..., Any] | None:
normal_method: Callable[..., Any] | None = getattr(
mw, "process_spider_output", None
)
async_method: Callable[..., Any] | None = getattr(
mw, "process_spider_output_async", None
)
if not async_method:
if normal_method and not isasyncgenfunction(normal_method):
logger.warning(
raise TypeError(
f"Middleware {global_object_name(mw.__class__)} doesn't support"
f" asynchronous spider output, this is deprecated and will stop"
f" working in a future version of Scrapy. The middleware should"
f" be updated to support it. Please see"
f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users"
f" for more information."
f" asynchronous spider output. Its process_spider_output() method"
f" should be an async generator function or it should additionally"
f" define a process_spider_output_async() method."
)
return normal_method
if not normal_method:
logger.error(
f"Middleware {global_object_name(mw.__class__)} has {methodname_async} "
f"without {methodname}, skipping this method."
f"Middleware {global_object_name(mw.__class__)} has"
f" process_spider_output_async() without process_spider_output(),"
f" skipping this method. Please rename it to process_spider_output()."
)
return None
if not isasyncgenfunction(async_method):
@ -431,8 +280,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
if isasyncgenfunction(normal_method):
logger.error(
f"{global_object_name(normal_method)} is an async "
f"generator function while {methodname_async} exists, "
f"skipping both methods."
f"generator function while process_spider_output_async() exists, "
f"skipping both methods. Please remove process_spider_output_async()."
)
return None
return normal_method, async_method
return async_method

28
scrapy/downloadermiddlewares/retry.py
View File

@ -12,7 +12,7 @@ once the spider has finished crawling all regular (non-failed) pages.
from __future__ import annotations
from logging import Logger, getLogger
from logging import Logger, getLevelName, getLogger
from typing import TYPE_CHECKING
from scrapy.exceptions import NotConfigured
@ -43,6 +43,7 @@ def get_retry_request(
max_retry_times: int | None = None,
priority_adjust: int | None = None,
logger: Logger = retry_logger,
give_up_log_level: int | str | None = None,
stats_base_key: str = "retry",
) -> Request | None:
"""
@ -51,14 +52,16 @@ def get_retry_request(
exhausted.
For example, in a :class:`~scrapy.Spider` callback, you could use it as
follows::
follows:
.. code-block:: python
def parse(self, response):
if not response.text:
new_request_or_none = get_retry_request(
response.request,
spider=self,
reason='empty',
reason="empty",
)
return new_request_or_none
@ -82,6 +85,10 @@ def get_retry_request(
*logger* is the logging.Logger object to be used when logging messages
*give_up_log_level* is the :ref:`logging level <levels>` used for the
message logged when a request exceeds its retries. See
:setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
*stats_base_key* is a string to be used as the base key for the
retry-related job stats
"""
@ -114,8 +121,16 @@ def get_retry_request(
stats.inc_value(f"{stats_base_key}/count")
stats.inc_value(f"{stats_base_key}/reason_count/{reason}")
return new_request
if give_up_log_level is None:
give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"]
if isinstance(give_up_log_level, str):
level = getLevelName(give_up_log_level)
if not isinstance(level, int):
raise ValueError(f"Invalid give-up log level: {give_up_log_level!r}")
give_up_log_level = level
stats.inc_value(f"{stats_base_key}/max_reached")
logger.error(
logger.log(
give_up_log_level,
"Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s",
{"request": request, "retry_times": retry_times, "reason": reason},
extra={"spider": spider},
@ -132,6 +147,7 @@ class RetryMiddleware:
self.max_retry_times = settings.getint("RETRY_TIMES")
self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}
self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"]
self.exceptions_to_retry = tuple(
load_object(x) if isinstance(x, str) else x
for x in settings.getlist("RETRY_EXCEPTIONS")
@ -175,6 +191,9 @@ class RetryMiddleware:
) -> Request | None:
max_retry_times = request.meta.get("max_retry_times", self.max_retry_times)
priority_adjust = request.meta.get("priority_adjust", self.priority_adjust)
give_up_log_level = request.meta.get(
"give_up_log_level", self.give_up_log_level
)
assert self.crawler.spider
return get_retry_request(
request,
@ -182,4 +201,5 @@ class RetryMiddleware:
spider=self.crawler.spider,
max_retry_times=max_retry_times,
priority_adjust=priority_adjust,
give_up_log_level=give_up_log_level,
)

13
scrapy/extensions/feedexport.py
View File

@ -306,12 +306,15 @@ class GCSFeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file: IO[bytes]) -> None:
file.seek(0)
from google.cloud.storage import Client # noqa: PLC0415
try:
from google.cloud.storage import Client # noqa: PLC0415
client = Client(project=self.project_id)
bucket = client.get_bucket(self.bucket_name)
blob = bucket.blob(self.blob_name)
blob.upload_from_file(file, predefined_acl=self.acl)
client = Client(project=self.project_id)
bucket = client.get_bucket(self.bucket_name)
blob = bucket.blob(self.blob_name)
blob.upload_from_file(file, predefined_acl=self.acl)
finally:
file.close()
class FTPFeedStorage(BlockingFeedStorage):

4
scrapy/extensions/httpcache.py
View File

@ -313,7 +313,9 @@ class FilesystemCacheStorage:
self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")
self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP")
# https://github.com/python/mypy/issues/10740
self._open: Callable[Concatenate[str | os.PathLike, str, ...], IO[bytes]] = (
self._open: Callable[
Concatenate[str | os.PathLike[str], str, ...], IO[bytes]
] = (
gzip.open if self.use_gzip else open # type: ignore[assignment]
)

10
scrapy/http/request/__init__.py
View File

@ -138,6 +138,7 @@ class Request(object_ref):
) -> None:
self._encoding: str = encoding # this one has to be set first
self.method: str = str(method).upper()
self._meta: dict[str, Any] | None = dict(meta) if meta else None
self._set_url(url)
self._set_body(body)
if not isinstance(priority, int):
@ -232,7 +233,6 @@ class Request(object_ref):
#: default. See :meth:`~scrapy.Spider.start`.
self.dont_filter: bool = dont_filter
self._meta: dict[str, Any] | None = dict(meta) if meta else None
self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else None
self._flags: list[str] | None = list(flags) if flags else None
@ -252,11 +252,17 @@ class Request(object_ref):
def url(self) -> str:
return self._url
def _url_is_verbatim(self) -> bool:
return bool(self._meta and self._meta.get("verbatim_url"))
def _set_url(self, url: str) -> None:
if not isinstance(url, str):
raise TypeError(f"Request url must be str, got {type(url).__name__}")
self._url = safe_url_string(url, self.encoding)
if self._url_is_verbatim():
self._url = url
else:
self._url = safe_url_string(url, self.encoding)
if (
"://" not in self._url

5
scrapy/http/response/__init__.py
View File

@ -20,7 +20,6 @@ if TYPE_CHECKING:
from collections.abc import Callable, Iterable, Mapping
from ipaddress import IPv4Address, IPv6Address
from twisted.internet.ssl import Certificate
from twisted.python.failure import Failure
# typing.Self requires Python 3.11
@ -77,7 +76,7 @@ class Response(object_ref):
body: bytes = b"",
flags: list[str] | None = None,
request: Request | None = None,
certificate: Certificate | None = None,
certificate: Any = None,
ip_address: IPv4Address | IPv6Address | None = None,
protocol: str | None = None,
):
@ -87,7 +86,7 @@ class Response(object_ref):
self._set_url(url)
self.request: Request | None = request
self._flags: list[str] | None = list(flags) if flags else None
self.certificate: Certificate | None = certificate
self.certificate: Any = certificate
self.ip_address: IPv4Address | IPv6Address | None = ip_address
self.protocol: str | None = protocol

2
scrapy/http/response/text.py
View File

@ -220,7 +220,7 @@ class TextResponse(Response):
def follow_all(
self,
urls: Iterable[str | Link] | parsel.SelectorList | None = None,
urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,
callback: CallbackT | None = None,
method: str = "GET",
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,

9
scrapy/middleware.py
View File

@ -50,11 +50,8 @@ class MiddlewareManager(ABC):
)
self.middlewares: tuple[Any, ...] = middlewares
# Only process_spider_output and process_spider_exception can be None.
# Only process_spider_output can be a tuple, and only until _async compatibility methods are removed.
self.methods: dict[str, deque[Callable | tuple[Callable, Callable] | None]] = (
defaultdict(deque)
)
self._mw_methods_requiring_spider: set[Callable] = set()
self.methods: dict[str, deque[Callable[..., Any] | None]] = defaultdict(deque)
self._mw_methods_requiring_spider: set[Callable[..., Any]] = set()
for mw in middlewares:
self._add_middleware(mw)
@ -119,7 +116,7 @@ class MiddlewareManager(ABC):
def _add_middleware(self, mw: Any) -> None: # noqa: B027
pass
def _check_mw_method_spider_arg(self, method: Callable) -> None:
def _check_mw_method_spider_arg(self, method: Callable[..., Any]) -> None:
if argument_is_required(method, "spider"):
warnings.warn(
f"{method.__qualname__}() requires a spider argument,"

77
scrapy/settings/__init__.py
View File

@ -16,10 +16,6 @@ from scrapy.utils.python import global_object_name
logger = getLogger(__name__)
# The key types are restricted in BaseSettings._get_key() to ones supported by JSON,
# see https://github.com/scrapy/scrapy/issues/5383.
_SettingsKey: TypeAlias = bool | float | int | str | None
if TYPE_CHECKING:
from types import ModuleType
@ -29,7 +25,7 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
_SettingsInput: TypeAlias = SupportsItems[_SettingsKey, Any] | str | None
_SettingsInput: TypeAlias = SupportsItems[str, Any] | str | None
SETTINGS_PRIORITIES: dict[str, int] = {
@ -80,7 +76,7 @@ class SettingsAttribute:
return f"<SettingsAttribute value={self.value!r} priority={self.priority}>"
class BaseSettings(MutableMapping[_SettingsKey, Any]):
class BaseSettings(MutableMapping[str, Any]):
"""
Instances of this class behave like dictionaries, but store priorities
along with their ``(key, value)`` pairs, and can be frozen (i.e. marked
@ -106,11 +102,11 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
def __init__(self, values: _SettingsInput = None, priority: int | str = "project"):
self.frozen: bool = False
self.attributes: dict[_SettingsKey, SettingsAttribute] = {}
self.attributes: dict[str, SettingsAttribute] = {}
if values:
self.update(values, priority)
def __getitem__(self, opt_name: _SettingsKey) -> Any:
def __getitem__(self, opt_name: str) -> Any:
if opt_name not in self:
return None
return self.attributes[opt_name].value
@ -118,7 +114,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
def __contains__(self, name: Any) -> bool:
return name in self.attributes
def add_to_list(self, name: _SettingsKey, item: Any) -> None:
def add_to_list(self, name: str, item: Any) -> None:
"""Append *item* to the :class:`list` setting with the specified *name*
if *item* is not already in that list.
@ -129,7 +125,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
if item not in value:
self.set(name, [*value, item], self.getpriority(name) or 0)
def remove_from_list(self, name: _SettingsKey, item: Any) -> None:
def remove_from_list(self, name: str, item: Any) -> None:
"""Remove *item* from the :class:`list` setting with the specified
*name*.
@ -143,7 +139,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).")
self.set(name, [v for v in value if v != item], self.getpriority(name) or 0)
def get(self, name: _SettingsKey, default: Any = None) -> Any: # pylint: disable=arguments-renamed
def get(self, name: str, default: Any = None) -> Any: # pylint: disable=arguments-renamed
"""
Get a setting value without affecting its original type.
@ -172,7 +168,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
return self[name] if self[name] is not None else default
def getbool(self, name: _SettingsKey, default: bool = False) -> bool:
def getbool(self, name: str, default: bool = False) -> bool:
"""
Get a setting value as a boolean.
@ -202,7 +198,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
"'True'/'False' and 'true'/'false'"
) from None
def getint(self, name: _SettingsKey, default: int = 0) -> int:
def getint(self, name: str, default: int = 0) -> int:
"""
Get a setting value as an int.
@ -214,7 +210,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
"""
return int(self.get(name, default))
def getfloat(self, name: _SettingsKey, default: float = 0.0) -> float:
def getfloat(self, name: str, default: float = 0.0) -> float:
"""
Get a setting value as a float.
@ -226,9 +222,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
"""
return float(self.get(name, default))
def getlist(
self, name: _SettingsKey, default: list[Any] | None = None
) -> list[Any]:
def getlist(self, name: str, default: list[Any] | None = None) -> list[Any]:
"""
Get a setting value as a list. If the setting original type is a list,
a copy of it will be returned. If it's a string it will be split by
@ -251,7 +245,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
return list(value)
def getdict(
self, name: _SettingsKey, default: dict[Any, Any] | None = None
self, name: str, default: dict[Any, Any] | None = None
) -> dict[Any, Any]:
"""
Get a setting value as a dictionary. If the setting original type is a
@ -275,7 +269,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
def getdictorlist(
self,
name: _SettingsKey,
name: str,
default: dict[Any, Any] | list[Any] | tuple[Any] | None = None,
) -> dict[Any, Any] | list[Any]:
"""Get a setting value as either a :class:`dict` or a :class:`list`.
@ -322,7 +316,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
)
return copy.deepcopy(value)
def getwithbase(self, name: _SettingsKey) -> BaseSettings:
def getwithbase(self, name: str) -> BaseSettings:
"""Get a composition of a dictionary-like setting and its ``_BASE``
counterpart.
@ -341,7 +335,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
compbs.update(self[name])
return compbs
def get_component_priority_dict_with_base(self, name: _SettingsKey) -> BaseSettings:
def get_component_priority_dict_with_base(self, name: str) -> BaseSettings:
"""Get a composition of a component priority dictionary setting and
its ``_BASE`` counterpart.
@ -389,7 +383,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
{restore_key(k): v for k, v in result.items() if v is not None}
)
def getpriority(self, name: _SettingsKey) -> int | None:
def getpriority(self, name: str) -> int | None:
"""
Return the current numerical priority value of a setting, or ``None`` if
the given ``name`` does not exist.
@ -414,7 +408,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
def replace_in_component_priority_dict(
self,
name: _SettingsKey,
name: str,
old_cls: type,
new_cls: type,
priority: int | None = None,
@ -453,12 +447,10 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
)
self.set(name, component_priority_dict, priority=self.getpriority(name) or 0)
def __setitem__(self, name: _SettingsKey, value: Any) -> None:
def __setitem__(self, name: str, value: Any) -> None:
self.set(name, value)
def set(
self, name: _SettingsKey, value: Any, priority: int | str = "project"
) -> None:
def set(self, name: str, value: Any, priority: int | str = "project") -> None:
"""
Store a key/value attribute with a given priority.
@ -487,7 +479,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
self.attributes[name].set(value, priority)
def set_in_component_priority_dict(
self, name: _SettingsKey, cls: type, priority: int | None
self, name: str, cls: type, priority: int | None
) -> None:
"""Set the *cls* component in the *name* :ref:`component priority
dictionary <component-priority-dictionaries>` setting with *priority*.
@ -512,7 +504,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
def setdefault( # pylint: disable=arguments-renamed
self,
name: _SettingsKey,
name: str,
default: Any = None,
priority: int | str = "project",
) -> Any:
@ -523,7 +515,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
return self.attributes[name].value
def setdefault_in_component_priority_dict(
self, name: _SettingsKey, cls: type, priority: int | None
self, name: str, cls: type, priority: int | None
) -> None:
"""Set the *cls* component in the *name* :ref:`component priority
dictionary <component-priority-dictionaries>` setting with *priority*
@ -592,7 +584,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
"""
self._assert_mutability()
if isinstance(values, str):
values = cast("dict[_SettingsKey, Any]", json.loads(values))
values = cast("dict[str, Any]", json.loads(values))
if values is not None:
if isinstance(values, BaseSettings):
for name, value in values.items():
@ -601,7 +593,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
for name, value in values.items():
self.set(name, value, priority)
def delete(self, name: _SettingsKey, priority: int | str = "project") -> None:
def delete(self, name: str, priority: int | str = "project") -> None:
if name not in self:
raise KeyError(name)
self._assert_mutability()
@ -609,7 +601,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
if priority >= cast("int", self.getpriority(name)):
del self.attributes[name]
def __delitem__(self, name: _SettingsKey) -> None:
def __delitem__(self, name: str) -> None:
self._assert_mutability()
del self.attributes[name]
@ -649,26 +641,19 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
copy.freeze()
return copy
def __iter__(self) -> Iterator[_SettingsKey]:
def __iter__(self) -> Iterator[str]:
return iter(self.attributes)
def __len__(self) -> int:
return len(self.attributes)
def _to_dict(self) -> dict[_SettingsKey, Any]:
def _to_dict(self) -> dict[str, Any]:
return {
self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
str(k): (v._to_dict() if isinstance(v, BaseSettings) else v)
for k, v in self.items()
}
def _get_key(self, key_value: Any) -> _SettingsKey:
return (
key_value
if isinstance(key_value, (bool, float, int, str, type(None)))
else str(key_value)
)
def copy_to_dict(self) -> dict[_SettingsKey, Any]:
def copy_to_dict(self) -> dict[str, Any]:
"""
Make a copy of current settings and convert to a dict.
@ -691,7 +676,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
else:
p.text(pformat(self.copy_to_dict()))
def pop(self, name: _SettingsKey, default: Any = __default) -> Any: # pylint: disable=arguments-renamed
def pop(self, name: str, default: Any = __default) -> Any: # pylint: disable=arguments-renamed
try:
value = self.attributes[name].value
except KeyError:
@ -735,7 +720,7 @@ def iter_default_settings() -> Iterable[tuple[str, Any]]:
def overridden_settings(
settings: Mapping[_SettingsKey, Any],
settings: Mapping[str, Any],
) -> Iterable[tuple[str, Any]]:
"""Return an iterable of the settings that have been overridden"""
for name, defvalue in iter_default_settings():

63
scrapy/settings/default_settings.py
View File

@ -16,22 +16,30 @@ Scrapy developers, if you add a setting here remember to:
import sys
from importlib import import_module
from pathlib import Path
from typing import Any
__all__ = [
"ADDONS",
"AJAXCRAWL_ENABLED",
"AJAXCRAWL_MAXSIZE",
"ASYNCIO_EVENT_LOOP",
"AUTOTHROTTLE_DEBUG",
"AUTOTHROTTLE_ENABLED",
"AUTOTHROTTLE_MAX_DELAY",
"AUTOTHROTTLE_START_DELAY",
"AUTOTHROTTLE_TARGET_CONCURRENCY",
"AWS_ACCESS_KEY_ID",
"AWS_ENDPOINT_URL",
"AWS_REGION_NAME",
"AWS_SECRET_ACCESS_KEY",
"AWS_SESSION_TOKEN",
"AWS_USE_SSL",
"AWS_VERIFY",
"BOT_NAME",
"CLOSESPIDER_ERRORCOUNT",
"CLOSESPIDER_ITEMCOUNT",
"CLOSESPIDER_PAGECOUNT",
"CLOSESPIDER_PAGECOUNT_NO_ITEM",
"CLOSESPIDER_TIMEOUT",
"CLOSESPIDER_TIMEOUT_NO_ITEM",
"COMMANDS_MODULE",
"COMPRESSION_ENABLED",
"CONCURRENT_ITEMS",
@ -63,9 +71,14 @@ __all__ = [
"DOWNLOAD_HANDLERS",
"DOWNLOAD_HANDLERS_BASE",
"DOWNLOAD_MAXSIZE",
"DOWNLOAD_SLOTS",
"DOWNLOAD_TIMEOUT",
"DOWNLOAD_TLS_MAX_VERSION",
"DOWNLOAD_TLS_MIN_VERSION",
"DOWNLOAD_VERIFY_CERTIFICATES",
"DOWNLOAD_WARNSIZE",
"DUPEFILTER_CLASS",
"DUPEFILTER_DEBUG",
"EDITOR",
"EXTENSIONS",
"EXTENSIONS_BASE",
@ -84,7 +97,9 @@ __all__ = [
"FEED_STORAGE_S3_ACL",
"FEED_STORE_EMPTY",
"FEED_TEMPDIR",
"FEED_URI",
"FEED_URI_PARAMS",
"FILES_STORE",
"FILES_STORE_GCS_ACL",
"FILES_STORE_S3_ACL",
"FORCE_CRAWLER_PROCESS",
@ -104,8 +119,12 @@ __all__ = [
"HTTPCACHE_IGNORE_SCHEMES",
"HTTPCACHE_POLICY",
"HTTPCACHE_STORAGE",
"HTTPERROR_ALLOWED_CODES",
"HTTPERROR_ALLOW_ALL",
"HTTPPROXY_AUTH_ENCODING",
"HTTPPROXY_ENABLED",
"HTTPX_HTTP2_ENABLED",
"IMAGES_STORE",
"IMAGES_STORE_GCS_ACL",
"IMAGES_STORE_S3_ACL",
"ITEM_PIPELINES",
@ -128,6 +147,8 @@ __all__ = [
"MAIL_HOST",
"MAIL_PASS",
"MAIL_PORT",
"MAIL_SSL",
"MAIL_TLS",
"MAIL_USER",
"MEMDEBUG_ENABLED",
"MEMDEBUG_NOTIFY",
@ -149,10 +170,12 @@ __all__ = [
"REDIRECT_MAX_TIMES",
"REDIRECT_PRIORITY_ADJUST",
"REFERER_ENABLED",
"REFERRER_POLICIES",
"REFERRER_POLICY",
"REQUEST_FINGERPRINTER_CLASS",
"RETRY_ENABLED",
"RETRY_EXCEPTIONS",
"RETRY_GIVE_UP_LOG_LEVEL",
"RETRY_HTTP_CODES",
"RETRY_PRIORITY_ADJUST",
"RETRY_TIMES",
@ -193,9 +216,6 @@ __all__ = [
ADDONS = {}
AJAXCRAWL_ENABLED = False
AJAXCRAWL_MAXSIZE = 32768
ASYNCIO_EVENT_LOOP = None
AUTOTHROTTLE_ENABLED = False
@ -204,12 +224,22 @@ AUTOTHROTTLE_MAX_DELAY = 60.0
AUTOTHROTTLE_START_DELAY = 5.0
AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
AWS_ACCESS_KEY_ID = None
AWS_SECRET_ACCESS_KEY = None
AWS_ENDPOINT_URL = None
AWS_REGION_NAME = None
AWS_SESSION_TOKEN = None
AWS_USE_SSL = None
AWS_VERIFY = None
BOT_NAME = "scrapybot"
CLOSESPIDER_ERRORCOUNT = 0
CLOSESPIDER_ITEMCOUNT = 0
CLOSESPIDER_PAGECOUNT = 0
CLOSESPIDER_TIMEOUT = 0
CLOSESPIDER_PAGECOUNT_NO_ITEM = 0
CLOSESPIDER_TIMEOUT_NO_ITEM = 0
COMMANDS_MODULE = ""
@ -262,15 +292,19 @@ DOWNLOAD_HANDLERS_BASE = {
DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m
DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m
DOWNLOAD_SLOTS = {}
DOWNLOAD_TIMEOUT = 180 # 3mins
DOWNLOAD_TLS_MAX_VERSION = None
DOWNLOAD_TLS_MIN_VERSION = None
DOWNLOAD_VERIFY_CERTIFICATES = False
DOWNLOADER = "scrapy.core.downloader.Downloader"
DOWNLOADER_CLIENTCONTEXTFACTORY = "SENTINEL"
DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT"
# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation:
DOWNLOADER_CLIENT_TLS_METHOD = "TLS"
DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
@ -297,6 +331,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
DOWNLOADER_STATS = True
DUPEFILTER_CLASS = "scrapy.dupefilters.RFPDupeFilter"
DUPEFILTER_DEBUG = False
EDITOR = "vi"
if sys.platform == "win32":
@ -347,8 +382,10 @@ FEED_STORAGE_FTP_ACTIVE = False
FEED_STORAGE_GCS_ACL = ""
FEED_STORAGE_S3_ACL = ""
FEED_TEMPDIR = None
FEED_URI = None
FEED_URI_PARAMS = None # a function to extend uri arguments
FILES_STORE = None
FILES_STORE_GCS_ACL = ""
FILES_STORE_S3_ACL = "private"
@ -373,9 +410,15 @@ HTTPCACHE_IGNORE_SCHEMES = ["file"]
HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy"
HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
HTTPERROR_ALLOW_ALL = False
HTTPERROR_ALLOWED_CODES = []
HTTPPROXY_ENABLED = True
HTTPPROXY_AUTH_ENCODING = "latin-1"
HTTPX_HTTP2_ENABLED = False
IMAGES_STORE = None
IMAGES_STORE_GCS_ACL = ""
IMAGES_STORE_S3_ACL = "private"
@ -416,6 +459,8 @@ MAIL_HOST = "localhost"
MAIL_PORT = 25
MAIL_USER = None
MAIL_PASS = None
MAIL_SSL = False
MAIL_TLS = False
MEMDEBUG_ENABLED = False # enable memory debugging
MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown
@ -446,6 +491,7 @@ REDIRECT_PRIORITY_ADJUST = +2
REFERER_ENABLED = True
REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy"
REFERRER_POLICIES = {}
REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter"
@ -464,6 +510,7 @@ RETRY_EXCEPTIONS = [
OSError,
"scrapy.core.downloader.handlers.http11.TunnelError",
]
RETRY_GIVE_UP_LOG_LEVEL = "ERROR"
RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
RETRY_PRIORITY_ADJUST = -1
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
@ -532,7 +579,7 @@ USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org
WARN_ON_GENERATOR_RETURN_VALUE = True
def __getattr__(name: str):
def __getattr__(name: str) -> Any:
if name == "CONCURRENT_REQUESTS_PER_IP":
import warnings # noqa: PLC0415
@ -545,4 +592,4 @@ def __getattr__(name: str):
)
return 0
raise AttributeError
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

11
scrapy/shell.py
View File

@ -28,11 +28,7 @@ from scrapy.spiders import Spider
from scrapy.utils.conf import get_config
from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console
from scrapy.utils.datatypes import SequenceExclude
from scrapy.utils.defer import (
_schedule_coro,
deferred_f_from_coro_f,
maybe_deferred_to_future,
)
from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.misc import load_object
from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop
from scrapy.utils.response import open_in_browser
@ -69,8 +65,8 @@ if TYPE_CHECKING:
# 3. When fetch() is called, it prepares a request and calls Shell._schedule()
# in the reactor thread (via threads.blockingCallFromThread()).
# 4. Shell._schedule() calls Shell._open_spider() (on the first call).
# 5. Shell._open_spider() calls engine.open_spider_async(close_if_idle=False)
# and engine._start_request_processing().
# 5. Shell._open_spider() calls
# engine.open_spider_async(close_if_idle=False).
# 6. Shell._schedule() calls engine.crawl(request), scheduling the request.
# 7. Shell._schedule() via _request_deferred() waits until the request callback
# is called. When it's called, the response becomes available.
@ -214,7 +210,6 @@ class Shell:
self.crawler.spider = spider
assert self.crawler.engine
await self.crawler.engine.open_spider_async(close_if_idle=False)
_schedule_coro(self.crawler.engine._start_request_processing())
self.spider = spider
def fetch(

4
scrapy/spiders/__init__.py
View File

@ -24,7 +24,7 @@ if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.http.request import CallbackT
from scrapy.settings import BaseSettings, _SettingsKey
from scrapy.settings import BaseSettings
from scrapy.utils.log import SpiderLoggerAdapter
@ -37,7 +37,7 @@ class Spider(object_ref):
"""
name: str
custom_settings: dict[_SettingsKey, Any] | None = None
custom_settings: dict[str, Any] | None = None
#: Start URLs. See :meth:`start`.
start_urls: list[str]

12
scrapy/spiders/crawl.py
View File

@ -12,6 +12,7 @@ import warnings
from collections.abc import AsyncIterator, Awaitable, Callable
from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Request, Response
from scrapy.link import Link
from scrapy.linkextractors import LinkExtractor
@ -46,7 +47,9 @@ def _identity_process_request(request: Request, response: Response) -> Request |
return request
def _get_method(method: Callable | str | None, spider: Spider) -> Callable | None:
def _get_method(
method: Callable[..., Any] | str | None, spider: Spider
) -> Callable[..., Any] | None:
if callable(method):
return method
if isinstance(method, str):
@ -218,4 +221,11 @@ class CrawlSpider(Spider):
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
spider = super().from_crawler(crawler, *args, **kwargs)
spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS")
if not spider._follow_links:
warnings.warn(
"The CRAWLSPIDER_FOLLOW_LINKS setting is deprecated."
" You can set follow=False in your rules to achieve the same effect.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
return spider

2
scrapy/squeues.py
View File

@ -26,7 +26,7 @@ if TYPE_CHECKING:
def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]:
class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc]
def __init__(self, path: str | PathLike, *args: Any, **kwargs: Any):
def __init__(self, path: str | PathLike[str], *args: Any, **kwargs: Any):
dirname = Path(path).parent
if not dirname.exists():
dirname.mkdir(parents=True, exist_ok=True)

7
scrapy/templates/project/module/items.py.tmpl
View File

@ -3,10 +3,11 @@
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html
import scrapy
from dataclasses import dataclass
class ${ProjectName}Item(scrapy.Item):
@dataclass
class ${ProjectName}Item:
# define the fields for your item here like:
# name = scrapy.Field()
# name: str | None = None
pass

2
scrapy/utils/_deps_compat.py
View File

@ -6,6 +6,8 @@ from twisted.python.versions import Version as TxVersion
TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0)
# changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506
TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
# lowerMaximumSecurityTo off-by-1, https://github.com/twisted/twisted/issues/10232
TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0)
PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)
# SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object

30
scrapy/utils/_download_handlers.py
View File

@ -2,8 +2,8 @@
from __future__ import annotations
from abc import ABC
from contextlib import contextmanager
from http.cookiejar import CookieJar
from typing import TYPE_CHECKING, Any
from twisted.internet.defer import CancelledError
@ -15,7 +15,6 @@ from twisted.web.client import ResponseFailed
from twisted.web.error import SchemeNotSupported
from scrapy import responsetypes
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.exceptions import (
CannotResolveHostError,
DownloadCancelledError,
@ -29,29 +28,24 @@ from scrapy.utils.log import logger
if TYPE_CHECKING:
from collections.abc import Iterator
from http.client import HTTPResponse
from http.cookiejar import Cookie
from ipaddress import IPv4Address, IPv6Address
from twisted.internet.ssl import Certificate
from urllib.request import Request as ULRequest
from scrapy import Request
from scrapy.crawler import Crawler
from scrapy.http import Headers, Response
class BaseHttpDownloadHandler(BaseDownloadHandler, ABC):
"""Base class for built-in HTTP download handlers."""
class NullCookieJar(CookieJar): # pragma: no cover
"""A CookieJar that rejects all cookies."""
def __init__(self, crawler: Crawler):
super().__init__(crawler)
self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE")
self._default_warnsize: int = crawler.settings.getint("DOWNLOAD_WARNSIZE")
self._fail_on_dataloss: bool = crawler.settings.getbool(
"DOWNLOAD_FAIL_ON_DATALOSS"
)
self._tls_verbose_logging: bool = crawler.settings.getbool(
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
)
self._fail_on_dataloss_warned: bool = False
def extract_cookies(self, response: HTTPResponse, request: ULRequest) -> None:
pass
def set_cookie(self, cookie: Cookie) -> None:
pass
@contextmanager
@ -103,7 +97,7 @@ def make_response(
headers: Headers,
body: bytes = b"",
flags: list[str] | None = None,
certificate: Certificate | None = None,
certificate: Any = None,
ip_address: IPv4Address | IPv6Address | None = None,
protocol: str | None = None,
stop_download: StopDownload | None = None,

2
scrapy/utils/asyncio.py
View File

@ -148,7 +148,7 @@ class AsyncioLoopingCall:
self._func: Callable[_P, _T] = func
self._args: tuple[Any, ...] = args
self._kwargs: dict[str, Any] = kwargs
self._task: asyncio.Task | None = None
self._task: asyncio.Task[None] | None = None
self.interval: float | None = None
self._start_time: float | None = None

4
scrapy/utils/conf.py
View File

@ -71,8 +71,8 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]:
def closest_scrapy_cfg(
path: str | os.PathLike = ".",
prevpath: str | os.PathLike | None = None,
path: str | os.PathLike[str] = ".",
prevpath: str | os.PathLike[str] | None = None,
) -> str:
"""Return the path to the closest scrapy.cfg file by traversing the current
directory and its parents

22
scrapy/utils/datatypes.py
View File

@ -28,7 +28,7 @@ _KT = TypeVar("_KT")
_VT = TypeVar("_VT")
class CaselessDict(dict):
class CaselessDict(dict): # type: ignore[type-arg]
__slots__ = ()
def __new__(cls, *args: Any, **kwargs: Any) -> Self:
@ -99,20 +99,20 @@ class CaselessDict(dict):
return dict.pop(self, self.normkey(key), *args)
class CaseInsensitiveDict(collections.UserDict):
class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):
"""A dict-like structure that accepts strings or bytes
as keys and allows case-insensitive lookups.
"""
def __init__(self, *args: Any, **kwargs: Any) -> None:
self._keys: dict = {}
self._keys: dict[str | bytes, Any] = {}
super().__init__(*args, **kwargs)
def __getitem__(self, key: AnyStr) -> Any:
def __getitem__(self, key: str | bytes) -> Any:
normalized_key = self._normkey(key)
return super().__getitem__(self._keys[normalized_key.lower()])
def __setitem__(self, key: AnyStr, value: Any) -> None:
def __setitem__(self, key: str | bytes, value: Any) -> None:
normalized_key = self._normkey(key)
try:
lower_key = self._keys[normalized_key.lower()]
@ -122,19 +122,19 @@ class CaseInsensitiveDict(collections.UserDict):
super().__setitem__(normalized_key, self._normvalue(value))
self._keys[normalized_key.lower()] = normalized_key
def __delitem__(self, key: AnyStr) -> None:
def __delitem__(self, key: str | bytes) -> None:
normalized_key = self._normkey(key)
stored_key = self._keys.pop(normalized_key.lower())
super().__delitem__(stored_key)
def __contains__(self, key: AnyStr) -> bool: # type: ignore[override]
def __contains__(self, key: str | bytes) -> bool: # type: ignore[override]
normalized_key = self._normkey(key)
return normalized_key.lower() in self._keys
def __repr__(self) -> str:
return f"<{self.__class__.__name__}: {super().__repr__()}>"
def _normkey(self, key: AnyStr) -> AnyStr:
def _normkey(self, key: str | bytes) -> str | bytes:
return key
def _normvalue(self, value: Any) -> Any:
@ -158,7 +158,7 @@ class LocalCache(OrderedDict[_KT, _VT]):
super().__setitem__(key, value)
class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):
"""
A weakref.WeakKeyDictionary implementation that uses LocalCache as its
underlying data structure, making it ordered and capable of being size-limited.
@ -172,9 +172,9 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
def __init__(self, limit: int | None = None):
super().__init__()
self.data: LocalCache = LocalCache(limit=limit)
self.data: LocalCache[_KT, _VT] = LocalCache(limit=limit)
def __setitem__(self, key: _KT, value: _VT) -> None:
def __setitem__(self, key: _KT, value: _VT | None) -> None:
# if raised, key is not weak-referenceable, skip caching
with contextlib.suppress(TypeError):
super().__setitem__(key, value)

2
scrapy/utils/defer.py
View File

@ -173,7 +173,7 @@ def parallel(
return DeferredList([coop.coiterate(work) for _ in range(count)])
class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_T]):
"""A class that wraps an async iterable into a normal iterator suitable
for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
it calls the callable directly in the callback, instead of providing a more

7
scrapy/utils/log.py
View File

@ -12,7 +12,7 @@ from twisted.python import log as twisted_log
from twisted.python.failure import Failure
import scrapy
from scrapy.settings import Settings, _SettingsKey
from scrapy.settings import Settings
from scrapy.utils.versions import get_versions
if TYPE_CHECKING:
@ -89,7 +89,7 @@ DEFAULT_LOGGING = {
def configure_logging(
settings: Settings | dict[_SettingsKey, Any] | None = None,
settings: Settings | dict[str, Any] | None = None,
install_root_handler: bool = True,
) -> None:
"""
@ -260,7 +260,8 @@ def logformatter_adapter(
return (level, message, args)
class SpiderLoggerAdapter(logging.LoggerAdapter):
# LoggerAdapter is only parameterized since Python 3.11
class SpiderLoggerAdapter(logging.LoggerAdapter): # type: ignore[type-arg]
def process(
self, msg: str, kwargs: MutableMapping[str, Any]
) -> tuple[str, MutableMapping[str, Any]]:

6
scrapy/utils/misc.py
View File

@ -28,7 +28,7 @@ if TYPE_CHECKING:
_ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
_ITER_T = TypeVar("_ITER_T", bound=dict | Item | str | bytes)
_ITER_T = TypeVar("_ITER_T", bound=dict[Any, Any] | Item | str | bytes)
_T = TypeVar("_T")
_T_co = TypeVar("_T_co", covariant=True)
_P = ParamSpec("_P")
@ -252,7 +252,9 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
yield node
_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
_generator_callbacks_cache: LocalWeakReferencedCache[Callable[..., Any], bool] = (
LocalWeakReferencedCache(limit=128)
)
def _returns_none(return_node: ast.Return) -> bool:

23
scrapy/utils/python.py
View File

@ -8,12 +8,14 @@ import gc
import inspect
import re
import sys
import warnings
import weakref
from collections.abc import AsyncIterator, Iterable, Mapping
from functools import partial, wraps
from itertools import chain
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.asyncgen import as_async_generator
if TYPE_CHECKING:
@ -294,12 +296,13 @@ else:
gc.collect()
class MutableChain(Iterable[_T]):
"""
Thin wrapper around itertools.chain, allowing to add iterables "in-place"
"""
class MutableChain(Iterable[_T]): # pragma: no cover
def __init__(self, *args: Iterable[_T]):
warnings.warn(
"MutableChain is deprecated and will be removed in a future Scrapy version.",
category=ScrapyDeprecationWarning,
stacklevel=2,
)
self.data: Iterator[_T] = chain.from_iterable(args)
def extend(self, *iterables: Iterable[_T]) -> None:
@ -356,3 +359,13 @@ def _looks_like_import_path(value: str) -> bool:
if any(part == "" for part in parts):
return False
return all(part.isidentifier() for part in parts)
def _iter_exc_causes(exc: BaseException) -> Iterable[BaseException]:
"""Iterate over the exception causes/contexts."""
seen: set[int] = set()
cur: BaseException | None = exc
while cur is not None and id(cur) not in seen:
seen.add(id(cur))
yield cur
cur = cur.__cause__ or cur.__context__

2
scrapy/utils/reactor.py
View File

@ -55,7 +55,7 @@ class CallLaterOnce(Generic[_T]):
self._a: tuple[Any, ...] = a
self._kw: dict[str, Any] = kw
self._call: CallLaterResult | None = None
self._deferreds: list[Deferred] = []
self._deferreds: list[Deferred[None]] = []
def schedule(self, delay: float = 0) -> None:
# circular import

17
scrapy/utils/request.py
View File

@ -28,7 +28,7 @@ if TYPE_CHECKING:
_fingerprint_cache: WeakKeyDictionary[
Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]
Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
] = WeakKeyDictionary()
@ -71,8 +71,10 @@ def fingerprint(
processed_include_headers = tuple(
to_bytes(h.lower()) for h in sorted(include_headers)
)
verbatim_url = bool(request.meta.get("verbatim_url"))
effective_keep_fragments = keep_fragments and not verbatim_url
cache = _fingerprint_cache.setdefault(request, {})
cache_key = (processed_include_headers, keep_fragments)
cache_key = (processed_include_headers, effective_keep_fragments, verbatim_url)
if cache_key not in cache:
# To decode bytes reliably (JSON does not support bytes), regardless of
# character encoding, we use bytes.hex()
@ -84,9 +86,13 @@ def fingerprint(
header_value.hex()
for header_value in request.headers.getlist(header)
]
if verbatim_url:
url = request.url
else:
url = canonicalize_url(request.url, keep_fragments=keep_fragments)
fingerprint_data = {
"method": to_unicode(request.method),
"url": canonicalize_url(request.url, keep_fragments=keep_fragments),
"url": url,
"body": (request.body or b"").hex(),
"headers": headers,
}
@ -108,8 +114,9 @@ class RequestFingerprinter:
(:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
<scrapy.Request.url>` and the values of :attr:`request.method
<scrapy.Request.method>` and :attr:`request.body
<scrapy.Request.body>`. It then generates an `SHA1
<https://en.wikipedia.org/wiki/SHA-1>`_ hash.
<scrapy.Request.body>`, unless :reqmeta:`verbatim_url` is true for that
request. It then generates an `SHA1 <https://en.wikipedia.org/wiki/SHA-1>`_
hash.
"""
@classmethod

3
scrapy/utils/response.py
View File

@ -97,8 +97,7 @@ def open_in_browser(
# XXX: this implementation is a bit dirty and could be improved
body = response.body
if isinstance(response, HtmlResponse):
if b"<base" not in body:
_remove_html_comments(body)
if b"<base" not in _remove_html_comments(body):
repl = rf'\g<0><base href="{response.url}">'
body = re.sub(rb"<head(?:[^<>]*?>)", to_bytes(repl), body, count=1)
ext = ".html"

4
scrapy/utils/signal.py
View File

@ -186,7 +186,9 @@ async def _send_catch_log_asyncio(
handlers: list[Awaitable[TypingAny]] = []
for receiver in liveReceivers(getAllReceivers(sender, signal)):
async def handler(receiver: Callable) -> tuple[Callable, TypingAny]:
async def handler(
receiver: Callable[..., Any],
) -> tuple[Callable[..., Any], TypingAny]:
result: TypingAny
try:
result = await ensure_awaitable(

113
scrapy/utils/ssl.py
View File

@ -2,30 +2,59 @@ from __future__ import annotations
import logging
import ssl
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, TypedDict, TypeVar
import OpenSSL._util as pyOpenSSLutil
import OpenSSL.SSL
import OpenSSL.version
from twisted.internet.ssl import CertificateOptions, TLSVersion
from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
from collections.abc import Callable
from OpenSSL.crypto import X509Name
from scrapy.settings import BaseSettings
logger = logging.getLogger(__name__)
_T = TypeVar("_T")
# common
def _get_tls_version_limit(
settings: BaseSettings, setting_name: str, converter: Callable[[str], _T]
) -> _T | None:
setting: str | None = settings[setting_name]
if setting is None:
return None
try:
return converter(setting)
except Exception as ex:
raise ValueError(f"Unknown {setting_name} value: {setting}") from ex
def _get_tls_version_limits(
settings: BaseSettings, converter: Callable[[str], _T]
) -> tuple[_T | None, _T | None]:
return (
_get_tls_version_limit(settings, "DOWNLOAD_TLS_MIN_VERSION", converter),
_get_tls_version_limit(settings, "DOWNLOAD_TLS_MAX_VERSION", converter),
)
# stdlib ssl module utils
# possible documented values for DOWNLOADER_CLIENT_TLS_METHOD
_STDLIB_PROTOCOL_MAP = {
"TLS": ssl.PROTOCOL_TLS_CLIENT,
"TLSv1.0": ssl.PROTOCOL_TLSv1,
"TLSv1.1": ssl.PROTOCOL_TLSv1_1,
"TLSv1.2": ssl.PROTOCOL_TLSv1_2,
_STDLIB_VERSION_MAP: dict[str, ssl.TLSVersion] = {
"TLSv1.0": ssl.TLSVersion.TLSv1,
"TLSv1.1": ssl.TLSVersion.TLSv1_1,
"TLSv1.2": ssl.TLSVersion.TLSv1_2,
"TLSv1.3": ssl.TLSVersion.TLSv1_3,
}
@ -35,13 +64,13 @@ def _make_ssl_context(settings: BaseSettings) -> ssl.SSLContext:
It's intended to be used in an HTTPS download handler.
"""
method_setting: str = settings["DOWNLOADER_CLIENT_TLS_METHOD"]
if method_setting not in _STDLIB_PROTOCOL_MAP:
raise ValueError(f"Unsupported TLS method: {method_setting}")
tls_min_ver, tls_max_ver = _get_tls_version_limits(
settings, _STDLIB_VERSION_MAP.__getitem__
)
ciphers_setting: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
verify_setting = settings.getbool("DOWNLOAD_VERIFY_CERTIFICATES")
ctx = ssl.SSLContext(_STDLIB_PROTOCOL_MAP[method_setting])
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
if verify_setting:
ctx.check_hostname = True
ctx.verify_mode = ssl.CERT_REQUIRED
@ -49,11 +78,27 @@ def _make_ssl_context(settings: BaseSettings) -> ssl.SSLContext:
else:
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
if tls_min_ver is not None:
ctx.minimum_version = tls_min_ver
if tls_max_ver is not None:
ctx.maximum_version = tls_max_ver
if ciphers_setting:
ctx.set_ciphers(ciphers_setting)
return ctx
def _make_insecure_ssl_ctx() -> ssl.SSLContext:
"""Create an SSL context that doesn't verify certificates.
Compared to :func:`~scrapy.utils.ssl._make_ssl_context` this is much more
simple.
"""
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
return ctx
def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None:
cipher = sslobj.cipher()
logger.debug(
@ -61,8 +106,11 @@ def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None:
f" using protocol {sslobj.version()},"
f" cipher {cipher[0] if cipher else None}"
)
# The peer certificate is unavailable on SSLObject unless peer
# certificate verification is enabled, which we don't want.
if cert := sslobj.getpeercert():
# Not available without certificate verification
logger.debug(
f'SSL connection certificate: issuer "{cert["issuer"]}", subject "{cert["subject"]}"'
)
# pyOpenSSL utils
@ -139,3 +187,42 @@ def _log_ssl_conn_debug_info(hostname: str, connection: OpenSSL.SSL.Connection)
key_info = get_temp_key_info(connection._ssl)
if key_info:
logger.debug("SSL temp key: %s", key_info)
# Twisted-specific
class _CertificateOptionsVersionKwargs(TypedDict, total=False):
lowerMaximumSecurityTo: TLSVersion
insecurelyLowerMinimumTo: TLSVersion
raiseMinimumTo: TLSVersion
def _get_cert_options_version_kwargs(
min_version: TLSVersion | None, max_version: TLSVersion | None
) -> _CertificateOptionsVersionKwargs:
"""Get TLS version kwargs for
:class:`~twisted.internet.ssl.CertificateOptions` for the given limits."""
result: _CertificateOptionsVersionKwargs = {}
if max_version:
if TWISTED_TLS_LIMITS_OFFBY1:
# lowerMaximumSecurityTo is treated as 1 version lower than the passed one
versions = list(TLSVersion.iterconstants())
max_index = versions.index(max_version)
if max_index + 1 >= len(versions):
raise ValueError(
f"Due to an error in Twisted < 26.4.0 cannot set the maximum TLS version to {max_version.name}"
)
max_version = versions[max_index + 1]
result["lowerMaximumSecurityTo"] = max_version
if min_version:
# We cannot pass both insecurelyLowerMinimumTo and raiseMinimumTo,
# so we need to know the direction.
# 1.0 in Twisted 22.8.0 and older, 1.2 in Twisted 22.10.0 and newer
default_min = CertificateOptions._defaultMinimumTLSVersion
if min_version < default_min:
result["insecurelyLowerMinimumTo"] = min_version
elif min_version > default_min:
result["raiseMinimumTo"] = min_version
return result

2
scrapy/utils/template.py
View File

@ -11,7 +11,7 @@ if TYPE_CHECKING:
from os import PathLike
def render_templatefile(path: str | PathLike, **kwargs: Any) -> None:
def render_templatefile(path: str | PathLike[str], **kwargs: Any) -> None:
path_obj = Path(path)
raw = path_obj.read_text("utf8")

4
scrapy/utils/trackref.py
View File

@ -30,7 +30,9 @@ if TYPE_CHECKING:
from typing_extensions import Self
live_refs: defaultdict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary)
live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict(
WeakKeyDictionary
)
class object_ref:

2
tests/AsyncCrawlerProcess/reactorless_custom_settings.py
View File

@ -23,7 +23,7 @@ class NoRequestsSpider(scrapy.Spider):
yield
def log_task_exception(task: Task) -> None:
def log_task_exception(task: Task[None]) -> None:
try:
task.result()
except Exception:

2
tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py
View File

@ -17,7 +17,7 @@ class AsyncioReactorSpider(scrapy.Spider):
}
def log_task_exception(task: Task) -> None:
def log_task_exception(task: Task[None]) -> None:
try:
task.result()
except Exception:

2
tests/mocks/dummydbm.py
View File

@ -4,7 +4,7 @@ from collections import defaultdict
from typing import Any
class DummyDB(dict):
class DummyDB(dict): # type: ignore[type-arg]
"""Provide dummy DBM-like interface."""
def close(self):

2
tests/mockserver/http.py
View File

@ -34,6 +34,7 @@ from .http_resources import (
ResponseHeadersResource,
SetCookie,
Status,
UriResource,
)
@ -84,6 +85,7 @@ class Root(resource.Resource):
self.putChild(b"duplicate-header", DuplicateHeaderResource())
self.putChild(b"response-headers", ResponseHeadersResource())
self.putChild(b"set-cookie", SetCookie())
self.putChild(b"uri", UriResource())
def getChild(self, path, request):
return self

14
tests/mockserver/http_base.py
View File

@ -106,6 +106,16 @@ def main_factory(
default=None,
help="SSL cipher string (optional)",
)
parser.add_argument(
"--tls-min-version",
default=None,
help="Minimum accepted TLS version (optional)",
)
parser.add_argument(
"--tls-max-version",
default=None,
help="Maximum accepted TLS version (optional)",
)
args = parser.parse_args()
context_factory_kw = {}
if args.keyfile:
@ -114,6 +124,10 @@ def main_factory(
context_factory_kw["certfile"] = args.certfile
if args.cipher_string:
context_factory_kw["cipher_string"] = args.cipher_string
if args.tls_min_version:
context_factory_kw["tls_min_version"] = args.tls_min_version
if args.tls_max_version:
context_factory_kw["tls_max_version"] = args.tls_max_version
context_factory = ssl_context_factory(**context_factory_kw)
https_port = reactor.listenSSL(0, factory, context_factory)

10
tests/mockserver/mitm_proxy.py
View File

@ -11,6 +11,9 @@ class MitmProxy:
auth_user = "scrapy"
auth_pass = "scrapy"
def __init__(self, mode: str | None = None) -> None:
self.mode = mode
def start(self) -> str:
script = """
import sys
@ -32,6 +35,8 @@ sys.exit(mitmdump())
"-s",
str(Path(__file__).with_name("mitm_proxy_addon.py")),
]
if self.mode:
args += ["--mode", self.mode]
self.proc: Popen[str] = Popen(
[
sys.executable,
@ -44,12 +49,13 @@ sys.exit(mitmdump())
text=True,
)
assert self.proc.stdout is not None
scheme = "socks5" if self.mode == "socks5" else "http"
line = ""
for line in self.proc.stdout:
m = re.search(r"listening at (?:http://)?([^:]+:\d+)", line)
m = re.search(r"listening at (?:\w+://)?([^:]+:\d+)", line)
if m:
host_port = m.group(1)
return f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
return f"{scheme}://{self.auth_user}:{self.auth_pass}@{host_port}"
self.stop()
raise RuntimeError(f"Failed to parse mitmdump output: {line}")

16
tests/mockserver/simple_https.py
View File

@ -21,11 +21,21 @@ class SimpleMockServer(BaseMockServer):
listen_http = False
module_name = "tests.mockserver.simple_https"
def __init__(self, keyfile: str, certfile: str, cipher_string: str | None):
def __init__(
self,
keyfile: str,
certfile: str,
*,
cipher_string: str | None = None,
tls_min_version: str | None = None,
tls_max_version: str | None = None,
):
super().__init__()
self.keyfile = keyfile
self.certfile = certfile
self.cipher_string = cipher_string or ""
self.tls_min_version = tls_min_version
self.tls_max_version = tls_max_version
def get_additional_args(self) -> list[str]:
args = [
@ -36,6 +46,10 @@ class SimpleMockServer(BaseMockServer):
]
if self.cipher_string is not None:
args.extend(["--cipher-string", self.cipher_string])
if self.tls_min_version is not None:
args.extend(["--tls-min-version", self.tls_min_version])
if self.tls_max_version is not None:
args.extend(["--tls-max-version", self.tls_max_version])
return args

9
tests/mockserver/utils.py
View File

@ -9,8 +9,10 @@ from OpenSSL import SSL
from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey
from twisted.internet.ssl import CertificateOptions, ContextFactory
from scrapy.core.downloader.tls import _TWISTED_VERSION_MAP
from scrapy.utils._deps_compat import PYOPENSSL_WANTS_X509_PKEY
from scrapy.utils.python import to_bytes
from scrapy.utils.ssl import _get_cert_options_version_kwargs
if TYPE_CHECKING:
from twisted.internet.interfaces import IOpenSSLContextFactory
@ -19,7 +21,10 @@ if TYPE_CHECKING:
def ssl_context_factory(
keyfile: str = "keys/localhost.key",
certfile: str = "keys/localhost.crt",
*,
cipher_string: str | None = None,
tls_min_version: str | None = None,
tls_max_version: str | None = None,
) -> IOpenSSLContextFactory:
keyfile_path = Path(__file__).parent.parent / keyfile
certfile_path = Path(__file__).parent.parent / certfile
@ -31,10 +36,14 @@ def ssl_context_factory(
cert = load_certificate(FILETYPE_PEM, certfile_path.read_bytes()) # type: ignore[assignment]
key = load_privatekey(FILETYPE_PEM, keyfile_path.read_bytes()) # type: ignore[assignment]
tls_min = _TWISTED_VERSION_MAP.get(tls_min_version) if tls_min_version else None
tls_max = _TWISTED_VERSION_MAP.get(tls_max_version) if tls_max_version else None
tls_version_kwargs = _get_cert_options_version_kwargs(tls_min, tls_max)
# https://github.com/twisted/twisted/issues/12638
factory: CertificateOptions = CertificateOptions(
privateKey=key, # type: ignore[arg-type]
certificate=cert, # type: ignore[arg-type]
**tls_version_kwargs,
)
if cipher_string:
ctx = factory.getContext()

12
tests/spiders.py
View File

@ -94,19 +94,19 @@ class DelaySpider(MetaSpider):
class LogSpider(MetaSpider):
name = "log_spider"
def log_debug(self, message: str, extra: dict | None = None):
def log_debug(self, message: str, extra: dict[str, Any] | None = None):
self.logger.debug(message, extra=extra)
def log_info(self, message: str, extra: dict | None = None):
def log_info(self, message: str, extra: dict[str, Any] | None = None):
self.logger.info(message, extra=extra)
def log_warning(self, message: str, extra: dict | None = None):
def log_warning(self, message: str, extra: dict[str, Any] | None = None):
self.logger.warning(message, extra=extra)
def log_error(self, message: str, extra: dict | None = None):
def log_error(self, message: str, extra: dict[str, Any] | None = None):
self.logger.error(message, extra=extra)
def log_critical(self, message: str, extra: dict | None = None):
def log_critical(self, message: str, extra: dict[str, Any] | None = None):
self.logger.critical(message, extra=extra)
def parse(self, response):
@ -417,7 +417,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
"""
name = "crawl_spider_with_parse_method"
custom_settings: dict = {
custom_settings: dict[str, Any] = {
"RETRY_HTTP_CODES": [], # no need to retry
}
rules = (Rule(LinkExtractor(), callback="parse", follow=True),)

2
tests/test_command_genspider.py
View File

@ -9,7 +9,7 @@ from tests.test_commands import TestProjectBase
from tests.utils.cmdline import call, proc
def find_in_file(filename: Path, regex: str) -> re.Match | None:
def find_in_file(filename: Path, regex: str) -> re.Match[str] | None:
"""Find first pattern occurrence in file"""
pattern = re.compile(regex)
with filename.open("r", encoding="utf-8") as f:

5
tests/test_command_shell.py
View File

@ -4,7 +4,7 @@ import os
import sys
from io import BytesIO
from pathlib import Path
from typing import TYPE_CHECKING, cast
from typing import TYPE_CHECKING
import pytest
from pexpect.popen_spawn import PopenSpawn
@ -151,8 +151,7 @@ class TestInteractiveShell:
env = os.environ.copy()
env["SCRAPY_PYTHON_SHELL"] = "python"
logfile = BytesIO()
# https://github.com/python/typeshed/issues/14915
p = PopenSpawn(args, env=cast("os._Environ", env), timeout=5)
p = PopenSpawn(args, env=env, timeout=5)
p.logfile_read = logfile
p.expect_exact("Available Scrapy objects")
p.sendline(f"fetch('{mockserver.url('/')}')")

2
tests/test_command_startproject.py
View File

@ -79,7 +79,7 @@ class TestStartprojectCommand:
def get_permissions_dict(
path: str | os.PathLike, renamings=None, ignore=None
path: str | os.PathLike[str], renamings=None, ignore=None
) -> dict[str, str]:
def get_permissions(path: Path) -> str:
return oct(path.stat().st_mode)

43
tests/test_core_downloader.py
View File

@ -14,7 +14,7 @@ from twisted.web import server, static
from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
from twisted.web.client import Response as TxResponse
from scrapy.core.downloader import Downloader, Slot
from scrapy.core.downloader import Downloader, Slot, tls
from scrapy.core.downloader.contextfactory import (
_load_context_factory_from_settings,
_ScrapyClientContextFactory,
@ -202,18 +202,37 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
def test_setting_none(self):
crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": None})
with pytest.raises(KeyError):
with (
pytest.warns(
ScrapyDeprecationWarning,
match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated",
),
pytest.raises(KeyError),
):
_load_context_factory_from_settings(crawler)
def test_setting_bad(self):
crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"})
with pytest.raises(KeyError):
with (
pytest.warns(
ScrapyDeprecationWarning,
match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated",
),
pytest.raises(KeyError),
):
_load_context_factory_from_settings(crawler)
@pytest.mark.filterwarnings(
r"ignore:Passing method to twisted\.internet\.ssl\.CertificateOptions:DeprecationWarning"
)
@coroutine_test
async def test_setting_explicit(self, server_url: str) -> None:
crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"})
client_context_factory = _load_context_factory_from_settings(crawler)
with pytest.warns(
ScrapyDeprecationWarning,
match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated",
):
client_context_factory = _load_context_factory_from_settings(crawler)
assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD
await self._assert_factory_works(server_url, client_context_factory)
@ -227,6 +246,9 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD
await self._assert_factory_works(server_url, client_context_factory)
@pytest.mark.filterwarnings(
r"ignore:Passing method to twisted\.internet\.ssl\.CertificateOptions:DeprecationWarning"
)
@coroutine_test
async def test_direct_init(self, server_url: str) -> None:
client_context_factory = _ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD)
@ -246,3 +268,16 @@ async def test_fetch_deprecated_spider_arg():
match=r"The fetch\(\) method of .+\.CustomDownloader requires a spider argument",
):
await crawler.crawl_async()
def test_deprecated_tls_module_names() -> None:
with pytest.warns(
ScrapyDeprecationWarning,
match="scrapy.core.downloader.tls.METHOD_TLS is deprecated",
):
assert tls.METHOD_TLS == "TLS"
with pytest.warns(
ScrapyDeprecationWarning,
match="scrapy.core.downloader.tls.openssl_methods is deprecated",
):
assert isinstance(tls.openssl_methods, dict)

790
tests/test_crawl.py
View File

File diff suppressed because it is too large Load Diff

268
tests/test_crawler.py
View File

@ -1,13 +1,13 @@
from __future__ import annotations
import asyncio
import logging
import re
import warnings
from collections.abc import Generator
from pathlib import Path
from typing import Any, cast
from typing import Any, ClassVar
import pytest
from twisted.internet.defer import Deferred
from zope.interface.exceptions import MultipleInvalid
import scrapy
@ -23,7 +23,7 @@ from scrapy.crawler import (
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.extensions.throttle import AutoThrottle
from scrapy.settings import Settings, default_settings
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
from scrapy.utils.log import (
_uninstall_scrapy_root_handler,
configure_logging,
@ -31,12 +31,14 @@ from scrapy.utils.log import (
)
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_reactor_settings
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from tests.utils.decorators import coroutine_test
BASE_SETTINGS: dict[str, Any] = {}
def get_raw_crawler(spidercls=None, settings_dict=None):
def get_raw_crawler(
spidercls: type[Spider] | None = None, settings_dict: dict[str, Any] | None = None
) -> Crawler:
"""get_crawler alternative that only calls the __init__ method of the
crawler."""
settings = Settings()
@ -46,14 +48,18 @@ def get_raw_crawler(spidercls=None, settings_dict=None):
class TestBaseCrawler:
def assertOptionIsDefault(self, settings: Settings, key: str) -> None:
@staticmethod
def assertOptionIsDefault(settings: Settings, key: str) -> None:
assert isinstance(settings, Settings)
assert settings[key] == getattr(default_settings, key)
class TestCrawler(TestBaseCrawler):
def test_populate_spidercls_settings(self):
spider_settings = {"TEST1": "spider", "TEST2": "spider"}
def test_populate_spidercls_settings(self) -> None:
spider_settings: dict[str, Any] = {
"TEST1": "spider",
"TEST2": "spider",
}
project_settings = {
**BASE_SETTINGS,
"TEST1": "project",
@ -76,47 +82,47 @@ class TestCrawler(TestBaseCrawler):
assert not settings.frozen
assert crawler.settings.frozen
def test_crawler_accepts_dict(self):
def test_crawler_accepts_dict(self) -> None:
crawler = get_crawler(DefaultSpider, {"foo": "bar"})
assert crawler.settings["foo"] == "bar"
self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
def test_crawler_accepts_None(self):
def test_crawler_accepts_None(self) -> None:
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
crawler = Crawler(DefaultSpider)
self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
def test_crawler_rejects_spider_objects(self):
def test_crawler_rejects_spider_objects(self) -> None:
with pytest.raises(ValueError, match="spidercls argument must be a class"):
Crawler(DefaultSpider())
@inline_callbacks_test
def test_crawler_crawl_twice_seq_unsupported(self):
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
yield crawler.crawl()
with pytest.raises(RuntimeError, match="more than once on the same instance"):
yield crawler.crawl()
Crawler(DefaultSpider()) # type: ignore[arg-type]
@coroutine_test
async def test_crawler_crawl_async_twice_seq_unsupported(self):
async def test_crawler_crawl_twice_seq_unsupported(self) -> None:
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
await maybe_deferred_to_future(crawler.crawl())
with pytest.raises(RuntimeError, match="more than once on the same instance"):
await maybe_deferred_to_future(crawler.crawl())
@coroutine_test
async def test_crawler_crawl_async_twice_seq_unsupported(self) -> None:
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
await crawler.crawl_async()
with pytest.raises(RuntimeError, match="more than once on the same instance"):
await crawler.crawl_async()
@inline_callbacks_test
def test_crawler_crawl_twice_parallel_unsupported(self):
@coroutine_test
async def test_crawler_crawl_twice_parallel_unsupported(self) -> None:
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
d1 = crawler.crawl()
d2 = crawler.crawl()
yield d1
await maybe_deferred_to_future(d1)
with pytest.raises(RuntimeError, match="Crawling already taking place"):
yield d2
await maybe_deferred_to_future(d2)
@pytest.mark.only_asyncio
@coroutine_test
async def test_crawler_crawl_async_twice_parallel_unsupported(self):
async def test_crawler_crawl_async_twice_parallel_unsupported(self) -> None:
crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
t1 = asyncio.create_task(crawler.crawl_async())
t2 = asyncio.create_task(crawler.crawl_async())
@ -124,12 +130,12 @@ class TestCrawler(TestBaseCrawler):
with pytest.raises(RuntimeError, match="Crawling already taking place"):
await t2
def test_get_addon(self):
def test_get_addon(self) -> None:
class ParentAddon:
pass
class TrackingAddon(ParentAddon):
instances = []
instances: ClassVar[list[TrackingAddon]] = []
def __init__(self):
TrackingAddon.instances.append(self)
@ -150,7 +156,7 @@ class TestCrawler(TestBaseCrawler):
addon = crawler.get_addon(TrackingAddon)
assert addon == expected
addon = crawler.get_addon(DefaultSpider)
addon = crawler.get_addon(DefaultSpider) # type: ignore[assignment]
assert addon is None
addon = crawler.get_addon(ParentAddon)
@ -162,19 +168,21 @@ class TestCrawler(TestBaseCrawler):
addon = crawler.get_addon(ChildAddon)
assert addon is None
@inline_callbacks_test
def test_get_downloader_middleware(self):
@coroutine_test
async def test_get_downloader_middleware(self) -> None:
class ParentDownloaderMiddleware:
pass
class TrackingDownloaderMiddleware(ParentDownloaderMiddleware):
instances = []
instances: ClassVar[list[TrackingDownloaderMiddleware]] = []
def __init__(self):
TrackingDownloaderMiddleware.instances.append(self)
class MySpider(Spider):
name = "myspider"
cls: ClassVar[type[Any]]
result: ClassVar[Any]
@classmethod
def from_crawler(cls, crawler):
@ -198,18 +206,18 @@ class TestCrawler(TestBaseCrawler):
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = TrackingDownloaderMiddleware
yield crawler.crawl()
await crawler.crawl_async()
assert len(TrackingDownloaderMiddleware.instances) == 1
assert MySpider.result == TrackingDownloaderMiddleware.instances[-1]
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = DefaultSpider
yield crawler.crawl()
await crawler.crawl_async()
assert MySpider.result is None
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ParentDownloaderMiddleware
yield crawler.crawl()
await crawler.crawl_async()
assert MySpider.result == TrackingDownloaderMiddleware.instances[-1]
class ChildDownloaderMiddleware(TrackingDownloaderMiddleware):
@ -217,16 +225,16 @@ class TestCrawler(TestBaseCrawler):
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ChildDownloaderMiddleware
yield crawler.crawl()
await crawler.crawl_async()
assert MySpider.result is None
def test_get_downloader_middleware_not_crawling(self):
def test_get_downloader_middleware_not_crawling(self) -> None:
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
with pytest.raises(RuntimeError):
crawler.get_downloader_middleware(DefaultSpider)
@inline_callbacks_test
def test_get_downloader_middleware_no_engine(self):
@coroutine_test
async def test_get_downloader_middleware_no_engine(self) -> None:
class MySpider(Spider):
name = "myspider"
@ -240,21 +248,23 @@ class TestCrawler(TestBaseCrawler):
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with pytest.raises(RuntimeError):
yield crawler.crawl()
await crawler.crawl_async()
@inline_callbacks_test
def test_get_extension(self):
@coroutine_test
async def test_get_extension(self) -> None:
class ParentExtension:
pass
class TrackingExtension(ParentExtension):
instances = []
instances: ClassVar[list[TrackingExtension]] = []
def __init__(self):
TrackingExtension.instances.append(self)
class MySpider(Spider):
name = "myspider"
cls: ClassVar[type[Any]]
result: ClassVar[Any]
@classmethod
def from_crawler(cls, crawler):
@ -278,18 +288,18 @@ class TestCrawler(TestBaseCrawler):
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = TrackingExtension
yield crawler.crawl()
await crawler.crawl_async()
assert len(TrackingExtension.instances) == 1
assert MySpider.result == TrackingExtension.instances[-1]
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = DefaultSpider
yield crawler.crawl()
await crawler.crawl_async()
assert MySpider.result is None
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ParentExtension
yield crawler.crawl()
await crawler.crawl_async()
assert MySpider.result == TrackingExtension.instances[-1]
class ChildExtension(TrackingExtension):
@ -297,16 +307,16 @@ class TestCrawler(TestBaseCrawler):
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ChildExtension
yield crawler.crawl()
await crawler.crawl_async()
assert MySpider.result is None
def test_get_extension_not_crawling(self):
def test_get_extension_not_crawling(self) -> None:
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
with pytest.raises(RuntimeError):
crawler.get_extension(DefaultSpider)
@inline_callbacks_test
def test_get_extension_no_engine(self):
@coroutine_test
async def test_get_extension_no_engine(self) -> None:
class MySpider(Spider):
name = "myspider"
@ -320,21 +330,23 @@ class TestCrawler(TestBaseCrawler):
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with pytest.raises(RuntimeError):
yield crawler.crawl()
await crawler.crawl_async()
@inline_callbacks_test
def test_get_item_pipeline(self):
@coroutine_test
async def test_get_item_pipeline(self) -> None:
class ParentItemPipeline:
pass
class TrackingItemPipeline(ParentItemPipeline):
instances = []
instances: ClassVar[list[TrackingItemPipeline]] = []
def __init__(self):
TrackingItemPipeline.instances.append(self)
class MySpider(Spider):
name = "myspider"
cls: ClassVar[type[Any]]
result: ClassVar[Any]
@classmethod
def from_crawler(cls, crawler):
@ -358,18 +370,18 @@ class TestCrawler(TestBaseCrawler):
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = TrackingItemPipeline
yield crawler.crawl()
await crawler.crawl_async()
assert len(TrackingItemPipeline.instances) == 1
assert MySpider.result == TrackingItemPipeline.instances[-1]
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = DefaultSpider
yield crawler.crawl()
await crawler.crawl_async()
assert MySpider.result is None
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ParentItemPipeline
yield crawler.crawl()
await crawler.crawl_async()
assert MySpider.result == TrackingItemPipeline.instances[-1]
class ChildItemPipeline(TrackingItemPipeline):
@ -377,16 +389,16 @@ class TestCrawler(TestBaseCrawler):
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ChildItemPipeline
yield crawler.crawl()
await crawler.crawl_async()
assert MySpider.result is None
def test_get_item_pipeline_not_crawling(self):
def test_get_item_pipeline_not_crawling(self) -> None:
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
with pytest.raises(RuntimeError):
crawler.get_item_pipeline(DefaultSpider)
@inline_callbacks_test
def test_get_item_pipeline_no_engine(self):
@coroutine_test
async def test_get_item_pipeline_no_engine(self) -> None:
class MySpider(Spider):
name = "myspider"
@ -400,21 +412,23 @@ class TestCrawler(TestBaseCrawler):
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with pytest.raises(RuntimeError):
yield crawler.crawl()
await crawler.crawl_async()
@inline_callbacks_test
def test_get_spider_middleware(self):
@coroutine_test
async def test_get_spider_middleware(self) -> None:
class ParentSpiderMiddleware:
pass
class TrackingSpiderMiddleware(ParentSpiderMiddleware):
instances = []
instances: ClassVar[list[TrackingSpiderMiddleware]] = []
def __init__(self):
TrackingSpiderMiddleware.instances.append(self)
class MySpider(Spider):
name = "myspider"
cls: ClassVar[type[Any]]
result: ClassVar[Any]
@classmethod
def from_crawler(cls, crawler):
@ -438,18 +452,18 @@ class TestCrawler(TestBaseCrawler):
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = TrackingSpiderMiddleware
yield crawler.crawl()
await crawler.crawl_async()
assert len(TrackingSpiderMiddleware.instances) == 1
assert MySpider.result == TrackingSpiderMiddleware.instances[-1]
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = DefaultSpider
yield crawler.crawl()
await crawler.crawl_async()
assert MySpider.result is None
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ParentSpiderMiddleware
yield crawler.crawl()
await crawler.crawl_async()
assert MySpider.result == TrackingSpiderMiddleware.instances[-1]
class ChildSpiderMiddleware(TrackingSpiderMiddleware):
@ -457,16 +471,16 @@ class TestCrawler(TestBaseCrawler):
crawler = get_raw_crawler(MySpider, settings)
MySpider.cls = ChildSpiderMiddleware
yield crawler.crawl()
await crawler.crawl_async()
assert MySpider.result is None
def test_get_spider_middleware_not_crawling(self):
def test_get_spider_middleware_not_crawling(self) -> None:
crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
with pytest.raises(RuntimeError):
crawler.get_spider_middleware(DefaultSpider)
@inline_callbacks_test
def test_get_spider_middleware_no_engine(self):
@coroutine_test
async def test_get_spider_middleware_no_engine(self) -> None:
class MySpider(Spider):
name = "myspider"
@ -480,22 +494,23 @@ class TestCrawler(TestBaseCrawler):
crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
with pytest.raises(RuntimeError):
yield crawler.crawl()
await crawler.crawl_async()
class TestSpiderSettings:
def test_spider_custom_settings(self):
def test_spider_custom_settings(self) -> None:
class MySpider(scrapy.Spider):
name = "spider"
custom_settings = {"AUTOTHROTTLE_ENABLED": True}
crawler = get_crawler(MySpider)
assert crawler.extensions
enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
assert AutoThrottle in enabled_exts
class TestCrawlerLogging:
def test_no_root_handler_installed(self):
def test_no_root_handler_installed(self) -> None:
handler = get_scrapy_root_handler()
if handler is not None:
logging.root.removeHandler(handler)
@ -507,7 +522,7 @@ class TestCrawlerLogging:
assert get_scrapy_root_handler() is None
@coroutine_test
async def test_spider_custom_settings_log_level(self, tmp_path):
async def test_spider_custom_settings_log_level(self, tmp_path: Path) -> None:
log_file = Path(tmp_path, "log.txt")
log_file.write_text("previous message\n", encoding="utf-8")
@ -535,9 +550,13 @@ class TestCrawlerLogging:
try:
configure_logging()
assert get_scrapy_root_handler().level == logging.DEBUG
handler = get_scrapy_root_handler()
assert handler is not None
assert handler.level == logging.DEBUG
crawler = get_crawler(MySpider)
assert get_scrapy_root_handler().level == logging.INFO
handler = get_scrapy_root_handler()
assert handler is not None
assert handler.level == logging.INFO
await crawler.crawl_async()
finally:
_uninstall_scrapy_root_handler()
@ -549,12 +568,13 @@ class TestCrawlerLogging:
assert "info message" in logged
assert "warning message" in logged
assert "error message" in logged
assert crawler.stats
assert crawler.stats.get_value("log_count/ERROR") == 1
assert crawler.stats.get_value("log_count/WARNING") == 1
assert info_count == 1
assert crawler.stats.get_value("log_count/DEBUG", 0) == 0
def test_spider_custom_settings_log_append(self, tmp_path):
def test_spider_custom_settings_log_append(self, tmp_path: Path) -> None:
log_file = Path(tmp_path, "log.txt")
log_file.write_text("previous message\n", encoding="utf-8")
@ -579,12 +599,12 @@ class TestCrawlerLogging:
class SpiderLoaderWithWrongInterface:
def unneeded_method(self):
def unneeded_method(self) -> None:
pass
class TestCrawlerRunner(TestBaseCrawler):
def test_spider_manager_verify_interface(self):
def test_spider_manager_verify_interface(self) -> None:
settings = Settings(
{
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
@ -593,18 +613,18 @@ class TestCrawlerRunner(TestBaseCrawler):
with pytest.raises(MultipleInvalid):
CrawlerRunner(settings)
def test_crawler_runner_accepts_dict(self):
def test_crawler_runner_accepts_dict(self) -> None:
runner = CrawlerRunner({"foo": "bar"})
assert runner.settings["foo"] == "bar"
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
def test_crawler_runner_accepts_None(self):
def test_crawler_runner_accepts_None(self) -> None:
runner = CrawlerRunner()
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
class TestAsyncCrawlerRunner(TestBaseCrawler):
def test_spider_manager_verify_interface(self):
def test_spider_manager_verify_interface(self) -> None:
settings = Settings(
{
"SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
@ -613,23 +633,23 @@ class TestAsyncCrawlerRunner(TestBaseCrawler):
with pytest.raises(MultipleInvalid):
AsyncCrawlerRunner(settings)
def test_crawler_runner_accepts_dict(self):
def test_crawler_runner_accepts_dict(self) -> None:
runner = AsyncCrawlerRunner({"foo": "bar"})
assert runner.settings["foo"] == "bar"
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
def test_crawler_runner_accepts_None(self):
def test_crawler_runner_accepts_None(self) -> None:
runner = AsyncCrawlerRunner()
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
class TestCrawlerProcess(TestBaseCrawler):
def test_crawler_process_accepts_dict(self):
def test_crawler_process_accepts_dict(self) -> None:
runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)
assert runner.settings["foo"] == "bar"
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
def test_crawler_process_accepts_None(self):
def test_crawler_process_accepts_None(self) -> None:
runner = CrawlerProcess(install_root_handler=False)
self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
@ -668,33 +688,35 @@ class NoRequestsSpider(scrapy.Spider):
@pytest.mark.requires_reactor # CrawlerRunner requires a reactor
class TestCrawlerRunnerHasSpider:
@staticmethod
def _runner() -> CrawlerRunnerBase:
@pytest.fixture
def runner(self) -> CrawlerRunnerBase:
return CrawlerRunner(get_reactor_settings())
@staticmethod
def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> Deferred[None]:
return cast("Deferred[None]", runner.crawl(spider))
async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:
await ensure_awaitable(runner.crawl(spider))
@inline_callbacks_test
def test_crawler_runner_bootstrap_successful(self):
runner = self._runner()
yield self._crawl(runner, NoRequestsSpider)
@coroutine_test
async def test_crawler_runner_bootstrap_successful(
self, runner: CrawlerRunnerBase
) -> None:
await self._crawl(runner, NoRequestsSpider)
assert not runner.bootstrap_failed
@inline_callbacks_test
def test_crawler_runner_bootstrap_successful_for_several(self):
runner = self._runner()
yield self._crawl(runner, NoRequestsSpider)
yield self._crawl(runner, NoRequestsSpider)
@coroutine_test
async def test_crawler_runner_bootstrap_successful_for_several(
self, runner: CrawlerRunnerBase
) -> None:
await self._crawl(runner, NoRequestsSpider)
await self._crawl(runner, NoRequestsSpider)
assert not runner.bootstrap_failed
@inline_callbacks_test
def test_crawler_runner_bootstrap_failed(self):
runner = self._runner()
@coroutine_test
async def test_crawler_runner_bootstrap_failed(
self, runner: CrawlerRunnerBase
) -> None:
try:
yield self._crawl(runner, ExceptionSpider)
await self._crawl(runner, ExceptionSpider)
except ValueError:
pass
else:
@ -702,25 +724,25 @@ class TestCrawlerRunnerHasSpider:
assert runner.bootstrap_failed
@inline_callbacks_test
def test_crawler_runner_bootstrap_failed_for_several(self):
runner = self._runner()
@coroutine_test
async def test_crawler_runner_bootstrap_failed_for_several(
self, runner: CrawlerRunnerBase
) -> None:
try:
yield self._crawl(runner, ExceptionSpider)
await self._crawl(runner, ExceptionSpider)
except ValueError:
pass
else:
pytest.fail("Exception should be raised from spider")
yield self._crawl(runner, NoRequestsSpider)
await self._crawl(runner, NoRequestsSpider)
assert runner.bootstrap_failed
@inline_callbacks_test
def test_crawler_runner_asyncio_enabled_true(
@coroutine_test
async def test_crawler_runner_asyncio_enabled_true(
self, reactor_pytest: str
) -> Generator[Deferred[Any], Any, None]:
) -> None:
if reactor_pytest != "asyncio":
runner = CrawlerRunner(
settings={
@ -731,7 +753,7 @@ class TestCrawlerRunnerHasSpider:
Exception,
match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
):
yield self._crawl(runner, NoRequestsSpider)
await self._crawl(runner, NoRequestsSpider)
else:
CrawlerRunner(
settings={
@ -742,15 +764,11 @@ class TestCrawlerRunnerHasSpider:
@pytest.mark.only_asyncio
class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
@staticmethod
def _runner() -> CrawlerRunnerBase:
@pytest.fixture
def runner(self) -> CrawlerRunnerBase:
return AsyncCrawlerRunner(get_reactor_settings())
@staticmethod
def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> Deferred[None]:
return deferred_from_coro(runner.crawl(spider))
def test_crawler_runner_asyncio_enabled_true(self):
def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override]
pytest.skip("This test is only for CrawlerRunner")
@ -762,7 +780,9 @@ class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
({"LOG_VERSIONS": []}, None),
],
)
def test_log_scrapy_info(settings, items, caplog):
def test_log_scrapy_info(
settings: dict[str, Any], items: list[str] | None, caplog: pytest.LogCaptureFixture
) -> None:
with caplog.at_level("INFO"):
CrawlerProcess(settings, install_root_handler=False)
assert (

100
tests/test_crawler_subprocess.py
View File

@ -52,7 +52,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
with the same file names and expectations.
"""
def test_simple(self):
def test_simple(self) -> None:
log = self.run_script("simple.py")
assert "Spider closed (finished)" in log
assert (
@ -61,7 +61,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
)
assert "is_reactorless(): False" in log
def test_multi(self):
def test_multi(self) -> None:
log = self.run_script("multi.py")
assert "Spider closed (finished)" in log
assert (
@ -70,7 +70,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
)
assert "ReactorAlreadyInstalledError" not in log
def test_reactor_default(self):
def test_reactor_default(self) -> None:
log = self.run_script("reactor_default.py")
assert "Spider closed (finished)" not in log
assert (
@ -78,7 +78,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
"(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
) in log
def test_asyncio_enabled_no_reactor(self):
def test_asyncio_enabled_no_reactor(self) -> None:
log = self.run_script("asyncio_enabled_no_reactor.py")
assert "Spider closed (finished)" in log
assert (
@ -87,7 +87,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
)
assert "RuntimeError" not in log
def test_asyncio_enabled_reactor(self):
def test_asyncio_enabled_reactor(self) -> None:
log = self.run_script("asyncio_enabled_reactor.py")
assert "Spider closed (finished)" in log
assert (
@ -100,7 +100,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
parse_version(w3lib_version) >= parse_version("2.0.0"),
reason="w3lib 2.0.0 and later do not allow invalid domains.",
)
def test_ipv6_default_name_resolver(self):
def test_ipv6_default_name_resolver(self) -> None:
log = self.run_script("default_name_resolver.py")
assert "Spider closed (finished)" in log
assert (
@ -112,7 +112,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
in log
)
def test_caching_hostname_resolver_ipv6(self):
def test_caching_hostname_resolver_ipv6(self) -> None:
log = self.run_script("caching_hostname_resolver_ipv6.py")
assert "Spider closed (finished)" in log
assert "scrapy.exceptions.CannotResolveHostError" not in log
@ -126,7 +126,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
assert "TimeoutError" not in log
assert "scrapy.exceptions.CannotResolveHostError" not in log
def test_twisted_reactor_asyncio(self):
def test_twisted_reactor_asyncio(self) -> None:
log = self.run_script("twisted_reactor_asyncio.py")
assert "Spider closed (finished)" in log
assert (
@ -134,7 +134,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
in log
)
def test_twisted_reactor_asyncio_custom_settings(self):
def test_twisted_reactor_asyncio_custom_settings(self) -> None:
log = self.run_script("twisted_reactor_custom_settings.py")
assert "Spider closed (finished)" in log
assert (
@ -142,7 +142,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
in log
)
def test_twisted_reactor_asyncio_custom_settings_same(self):
def test_twisted_reactor_asyncio_custom_settings_same(self) -> None:
log = self.run_script("twisted_reactor_custom_settings_same.py")
assert "Spider closed (finished)" in log
assert (
@ -151,7 +151,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
)
@pytest.mark.requires_uvloop
def test_custom_loop_asyncio(self):
def test_custom_loop_asyncio(self) -> None:
log = self.run_script("asyncio_custom_loop.py")
assert "Spider closed (finished)" in log
assert (
@ -161,7 +161,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
assert "Using asyncio event loop: uvloop.Loop" in log
@pytest.mark.requires_uvloop
def test_custom_loop_asyncio_deferred_signal(self):
def test_custom_loop_asyncio_deferred_signal(self) -> None:
log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
assert "Spider closed (finished)" in log
assert (
@ -172,7 +172,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
assert "async pipeline opened!" in log
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_same_loop(self):
def test_asyncio_enabled_reactor_same_loop(self) -> None:
log = self.run_script("asyncio_enabled_reactor_same_loop.py")
assert "Spider closed (finished)" in log
assert (
@ -182,7 +182,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
assert "Using asyncio event loop: uvloop.Loop" in log
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_different_loop(self):
def test_asyncio_enabled_reactor_different_loop(self) -> None:
log = self.run_script("asyncio_enabled_reactor_different_loop.py")
assert "Spider closed (finished)" not in log
assert (
@ -190,7 +190,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
"setting (uvloop.Loop)"
) in log
def test_default_loop_asyncio_deferred_signal(self):
def test_default_loop_asyncio_deferred_signal(self) -> None:
log = self.run_script("asyncio_deferred_signal.py")
assert "Spider closed (finished)" in log
assert (
@ -200,7 +200,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
assert "Using asyncio event loop: uvloop.Loop" not in log
assert "async pipeline opened!" in log
def test_args_change_settings(self):
def test_args_change_settings(self) -> None:
log = self.run_script("args_settings.py")
assert "Spider closed (finished)" in log
assert "The value of FOO is 42" in log
@ -243,7 +243,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
def script_dir(self) -> Path:
return self.get_script_dir("CrawlerProcess")
def test_reactor_default_twisted_reactor_select(self):
def test_reactor_default_twisted_reactor_select(self) -> None:
log = self.run_script("reactor_default_twisted_reactor_select.py")
if platform.system() in ["Windows", "Darwin"]:
# The goal of this test function is to test that, when a reactor is
@ -264,7 +264,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
"(twisted.internet.selectreactor.SelectReactor)"
) in log
def test_reactor_select(self):
def test_reactor_select(self) -> None:
log = self.run_script("reactor_select.py")
assert "Spider closed (finished)" not in log
assert (
@ -272,12 +272,12 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
"(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
) in log
def test_reactor_select_twisted_reactor_select(self):
def test_reactor_select_twisted_reactor_select(self) -> None:
log = self.run_script("reactor_select_twisted_reactor_select.py")
assert "Spider closed (finished)" in log
assert "ReactorAlreadyInstalledError" not in log
def test_reactor_select_subclass_twisted_reactor_select(self):
def test_reactor_select_subclass_twisted_reactor_select(self) -> None:
log = self.run_script("reactor_select_subclass_twisted_reactor_select.py")
assert "Spider closed (finished)" not in log
assert (
@ -285,7 +285,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
"(twisted.internet.selectreactor.SelectReactor)"
) in log
def test_twisted_reactor_select(self):
def test_twisted_reactor_select(self) -> None:
log = self.run_script("twisted_reactor_select.py")
assert "Spider closed (finished)" in log
assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log
@ -293,12 +293,12 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
@pytest.mark.skipif(
platform.system() == "Windows", reason="PollReactor is not supported on Windows"
)
def test_twisted_reactor_poll(self):
def test_twisted_reactor_poll(self) -> None:
log = self.run_script("twisted_reactor_poll.py")
assert "Spider closed (finished)" in log
assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log
def test_twisted_reactor_asyncio_custom_settings_conflict(self):
def test_twisted_reactor_asyncio_custom_settings_conflict(self) -> None:
log = self.run_script("twisted_reactor_custom_settings_conflict.py")
assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log
assert (
@ -306,7 +306,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
in log
)
def test_reactorless(self):
def test_reactorless(self) -> None:
log = self.run_script("reactorless.py")
assert (
"RuntimeError: CrawlerProcess doesn't support TWISTED_REACTOR_ENABLED=False"
@ -319,7 +319,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
def script_dir(self) -> Path:
return self.get_script_dir("AsyncCrawlerProcess")
def test_twisted_reactor_custom_settings_select(self):
def test_twisted_reactor_custom_settings_select(self) -> None:
log = self.run_script("twisted_reactor_custom_settings_select.py")
assert "Spider closed (finished)" not in log
assert (
@ -329,7 +329,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
) in log
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_same_loop(self):
def test_asyncio_enabled_reactor_same_loop(self) -> None:
log = self.run_script("asyncio_custom_loop_custom_settings_same.py")
assert "Spider closed (finished)" in log
assert (
@ -339,7 +339,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
assert "Using asyncio event loop: uvloop.Loop" in log
@pytest.mark.requires_uvloop
def test_asyncio_enabled_reactor_different_loop(self):
def test_asyncio_enabled_reactor_different_loop(self) -> None:
log = self.run_script("asyncio_custom_loop_custom_settings_different.py")
assert "Spider closed (finished)" not in log
assert (
@ -347,7 +347,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
"setting (uvloop.Loop)"
) in log
def test_reactorless_simple(self):
def test_reactorless_simple(self) -> None:
log = self.run_script("reactorless_simple.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
@ -356,7 +356,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2
assert log.count("WARNING: ") == 2
def test_reactorless_custom_settings(self):
def test_reactorless_custom_settings(self) -> None:
"""Setting TWISTED_REACTOR_ENABLED=False in spider settings is not
currently supported, AsyncCrawlerProcess will install a reactor in this
case.
@ -368,7 +368,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
in log
)
def test_reactorless_datauri(self):
def test_reactorless_datauri(self) -> None:
log = self.run_script("reactorless_datauri.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
@ -378,13 +378,13 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2
assert log.count("WARNING: ") == 2
def test_reactorless_import_hook(self):
def test_reactorless_import_hook(self) -> None:
log = self.run_script("reactorless_import_hook.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "ImportError: Import of twisted.internet.reactor is forbidden" in log
def test_reactorless_telnetconsole_default(self):
def test_reactorless_telnetconsole_default(self) -> None:
"""By default TWISTED_REACTOR_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False."""
log = self.run_script("reactorless_simple.py") # no need for a separate script
assert "Not using a Twisted reactor" in log
@ -392,7 +392,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
assert "The TelnetConsole extension requires a Twisted reactor" not in log
assert "scrapy.extensions.telnet.TelnetConsole" not in log
def test_reactorless_telnetconsole_disabled(self):
def test_reactorless_telnetconsole_disabled(self) -> None:
"""Explicit TELNETCONSOLE_ENABLED=False, there are no warnings."""
log = self.run_script("reactorless_telnetconsole_disabled.py")
assert "Not using a Twisted reactor" in log
@ -400,14 +400,14 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
assert "The TelnetConsole extension requires a Twisted reactor" not in log
assert "scrapy.extensions.telnet.TelnetConsole" not in log
def test_reactorless_telnetconsole_enabled(self):
def test_reactorless_telnetconsole_enabled(self) -> None:
"""Explicit TELNETCONSOLE_ENABLED=True, the user gets a warning."""
log = self.run_script("reactorless_telnetconsole_enabled.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
assert "The TelnetConsole extension requires a Twisted reactor" in log
def test_reactorless_reactor(self):
def test_reactorless_reactor(self) -> None:
log = self.run_script("reactorless_reactor.py")
assert (
"RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed"
@ -427,7 +427,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
with the same file names and expectations.
"""
def test_simple(self):
def test_simple(self) -> None:
log = self.run_script("simple.py")
assert "Spider closed (finished)" in log
assert (
@ -436,7 +436,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
)
assert "is_reactorless(): False" in log
def test_multi_parallel(self):
def test_multi_parallel(self) -> None:
log = self.run_script("multi_parallel.py")
assert "Spider closed (finished)" in log
assert (
@ -449,7 +449,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
re.DOTALL,
)
def test_multi_seq(self):
def test_multi_seq(self) -> None:
log = self.run_script("multi_seq.py")
assert "Spider closed (finished)" in log
assert (
@ -463,7 +463,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
)
@pytest.mark.requires_uvloop
def test_custom_loop_same(self):
def test_custom_loop_same(self) -> None:
log = self.run_script("custom_loop_same.py")
assert "Spider closed (finished)" in log
assert (
@ -473,7 +473,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
assert "Using asyncio event loop: uvloop.Loop" in log
@pytest.mark.requires_uvloop
def test_custom_loop_different(self):
def test_custom_loop_different(self) -> None:
log = self.run_script("custom_loop_different.py")
assert "Spider closed (finished)" not in log
assert (
@ -481,7 +481,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
"setting (uvloop.Loop)"
) in log
def test_no_reactor(self):
def test_no_reactor(self) -> None:
log = self.run_script("no_reactor.py")
assert "Spider closed (finished)" not in log
assert (
@ -495,7 +495,7 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
def script_dir(self) -> Path:
return self.get_script_dir("CrawlerRunner")
def test_explicit_default_reactor(self):
def test_explicit_default_reactor(self) -> None:
log = self.run_script("explicit_default_reactor.py")
assert "Spider closed (finished)" in log
assert (
@ -503,14 +503,14 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
not in log
)
def test_response_ip_address(self):
def test_response_ip_address(self) -> None:
log = self.run_script("ip_address.py")
assert "INFO: Spider closed (finished)" in log
assert "INFO: Host: not.a.real.domain" in log
assert "INFO: Type: <class 'ipaddress.IPv4Address'>" in log
assert "INFO: IP address: 127.0.0.1" in log
def test_change_default_reactor(self):
def test_change_default_reactor(self) -> None:
log = self.run_script("change_reactor.py")
assert (
"DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
@ -518,7 +518,7 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
)
assert "DEBUG: Using asyncio event loop" in log
def test_reactorless(self):
def test_reactorless(self) -> None:
log = self.run_script("reactorless.py")
assert (
"RuntimeError: CrawlerRunner doesn't support TWISTED_REACTOR_ENABLED=False"
@ -531,7 +531,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
def script_dir(self) -> Path:
return self.get_script_dir("AsyncCrawlerRunner")
def test_simple_default_reactor(self):
def test_simple_default_reactor(self) -> None:
log = self.run_script("simple_default_reactor.py")
assert "Spider closed (finished)" not in log
assert (
@ -539,7 +539,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
"AsyncCrawlerRunner requires that the installed Twisted reactor"
) in log
def test_reactorless_simple(self):
def test_reactorless_simple(self) -> None:
log = self.run_script("reactorless_simple.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
@ -548,7 +548,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2
assert log.count("WARNING: ") == 2
def test_reactorless_custom_settings(self):
def test_reactorless_custom_settings(self) -> None:
"""Setting TWISTED_REACTOR_ENABLED=False in spider settings is not
currently supported, AsyncCrawlerRunner will expect a reactor installed
by the user.
@ -557,7 +557,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
assert "Spider closed (finished)" not in log
assert "We expected a Twisted reactor to be installed but it isn't." in log
def test_reactorless_datauri(self):
def test_reactorless_datauri(self) -> None:
log = self.run_script("reactorless_datauri.py")
assert "Not using a Twisted reactor" in log
assert "Spider closed (finished)" in log
@ -567,7 +567,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2
assert log.count("WARNING: ") == 2
def test_reactorless_reactor(self):
def test_reactorless_reactor(self) -> None:
log = self.run_script("reactorless_reactor.py")
assert (
"RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed"

6
tests/test_dependencies.py
View File

@ -15,14 +15,14 @@ class TestScrapyUtils:
See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011
"""
if not os.environ.get("_SCRAPY_PINNED", None):
pytest.skip("Not in a pinned environment")
if not os.environ.get("_SCRAPY_MIN", None):
pytest.skip("Not in a min environment")
tox_config_file_path = Path(__file__).parent / ".." / "tox.ini"
config_parser = ConfigParser()
config_parser.read(tox_config_file_path)
pattern = r"Twisted==([\d.]+)"
match = re.search(pattern, config_parser["pinned"]["deps"])
match = re.search(pattern, config_parser["min"]["deps"])
pinned_twisted_version_string = match[1]
assert twisted_version.short() == pinned_twisted_version_string

76
tests/test_downloader_handler_httpx.py
View File

@ -3,11 +3,17 @@
from __future__ import annotations
import sys
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, ClassVar
import pytest
from scrapy import Request
from scrapy.core.downloader.handlers._httpx import (
HAS_HTTP2,
HAS_SOCKS,
HttpxDownloadHandler,
)
from scrapy.exceptions import DownloadFailedError
from tests.test_downloader_handlers_http_base import (
TestHttpBase,
TestHttpProxyBase,
@ -15,9 +21,11 @@ from tests.test_downloader_handlers_http_base import (
TestHttpsCustomCiphersBase,
TestHttpsInvalidDNSIdBase,
TestHttpsInvalidDNSPatternBase,
TestHttpsTLSVersionBase,
TestHttpsWrongHostnameBase,
TestHttpWithCrawlerBase,
TestMitmProxyBase,
TestRealWebsiteBase,
TestSimpleHttpsBase,
)
from tests.utils.decorators import coroutine_test
@ -35,11 +43,6 @@ pytest.importorskip("httpx")
class HttpxDownloadHandlerMixin:
@property
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
# the import will fail if httpx is not installed
from scrapy.core.downloader.handlers._httpx import ( # noqa: PLC0415
HttpxDownloadHandler,
)
return HttpxDownloadHandler
@property
@ -72,30 +75,40 @@ class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):
assert "DOWNLOAD_BIND_ADDRESS specifies a port (12345)" in caplog.text
assert "Ignoring the port" in caplog.text
@coroutine_test
async def test_unsupported_proxy(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
) -> None:
meta = {"proxy": "127.0.0.2"}
request = Request(mockserver.url("/text"), meta=meta)
async with self.get_dh() as download_handler:
response = await download_handler.download_request(request)
assert response.body == b"Works"
assert (
"The 'proxy' request meta key is not supported by HttpxDownloadHandler"
in caplog.text
)
class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase):
handler_supports_bindaddress_meta = False
tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher"
@pytest.mark.skip(reason="The check is Twisted-specific")
def test_verify_certs_deprecated(self):
def test_verify_certs_deprecated(self) -> None: # type: ignore[override]
pass
@pytest.mark.skipif(not HAS_HTTP2, reason="No HTTP/2 support in HttpxDownloadHandler")
class TestHttp2(TestHttps):
http2 = True
handler_supports_http2_dataloss = False
default_handler_settings: ClassVar[dict[str, Any]] = {
"HTTPX_HTTP2_ENABLED": True,
}
@coroutine_test
async def test_protocol(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/host", is_secure=self.is_secure))
async with self.get_dh() as download_handler:
response = await download_handler.download_request(request)
assert response.protocol == "HTTP/2"
@coroutine_test
async def test_data_loss_handling(self, mockserver: MockServer) -> None:
request = Request(mockserver.url("/broken", is_secure=self.is_secure))
async with self.get_dh() as download_handler:
with pytest.raises(DownloadFailedError):
await download_handler.download_request(request)
class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):
pass
@ -118,6 +131,10 @@ class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBa
pass
class TestHttpsTLSVersion(HttpxDownloadHandlerMixin, TestHttpsTLSVersionBase):
pass
class TestHttpWithCrawler(HttpxDownloadHandlerMixin, TestHttpWithCrawlerBase):
pass
@ -125,23 +142,20 @@ class TestHttpWithCrawler(HttpxDownloadHandlerMixin, TestHttpWithCrawlerBase):
class TestHttpsWithCrawler(TestHttpWithCrawler):
is_secure = True
@pytest.mark.skip(reason="response.certificate is not implemented")
@coroutine_test
async def test_response_ssl_certificate(self, mockserver: MockServer) -> None:
pass
@pytest.mark.skip(reason="Proxy support is not implemented yet")
class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
pass
expected_http_proxy_request_body = b"http://example.com/"
@pytest.mark.skip(reason="Proxy support is not implemented yet")
class TestHttpsProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
class TestHttpsProxy(TestHttpProxy):
is_secure = True
@pytest.mark.skip(reason="Proxy support is not implemented yet")
@pytest.mark.requires_mitmproxy
class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
handler_supports_socks = HAS_SOCKS
@pytest.mark.requires_internet
class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase):
pass

14
tests/test_downloader_handler_twisted_http11.py
View File

@ -2,6 +2,7 @@
from __future__ import annotations
import sys
from typing import TYPE_CHECKING, Any
import pytest
@ -17,9 +18,11 @@ from tests.test_downloader_handlers_http_base import (
TestHttpsCustomCiphersBase,
TestHttpsInvalidDNSIdBase,
TestHttpsInvalidDNSPatternBase,
TestHttpsTLSVersionBase,
TestHttpsWrongHostnameBase,
TestHttpWithCrawlerBase,
TestMitmProxyBase,
TestRealWebsiteBase,
TestSimpleHttpsBase,
)
@ -81,6 +84,10 @@ class TestHttpsCustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersB
pass
class TestHttpsTLSVersion(HTTP11DownloadHandlerMixin, TestHttpsTLSVersionBase):
pass
class TestHttpWithCrawler(HTTP11DownloadHandlerMixin, TestHttpWithCrawlerBase):
pass
@ -103,3 +110,10 @@ class TestHttpsProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):
class TestMitmProxy(HTTP11DownloadHandlerMixin, TestMitmProxyBase):
# not implemented
handler_supports_tls_in_tls = False
@pytest.mark.requires_internet
class TestRealWebsite(HTTP11DownloadHandlerMixin, TestRealWebsiteBase):
@property
def platform_cert_store_works(self) -> bool:
return sys.platform != "win32"

14
tests/test_downloader_handler_twisted_http2.py
View File

@ -2,6 +2,7 @@
from __future__ import annotations
import sys
from typing import TYPE_CHECKING, Any
import pytest
@ -18,9 +19,11 @@ from tests.test_downloader_handlers_http_base import (
TestHttpsCustomCiphersBase,
TestHttpsInvalidDNSIdBase,
TestHttpsInvalidDNSPatternBase,
TestHttpsTLSVersionBase,
TestHttpsWrongHostnameBase,
TestHttpWithCrawlerBase,
TestMitmProxyBase,
TestRealWebsiteBase,
)
from tests.utils.decorators import coroutine_test
@ -171,6 +174,10 @@ class TestHttp2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase)
pass
class TestHttp2TLSVersion(H2DownloadHandlerMixin, TestHttpsTLSVersionBase):
pass
class TestHttp2WithCrawler(H2DownloadHandlerMixin, TestHttpWithCrawlerBase):
is_secure = True
@ -196,3 +203,10 @@ class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
@pytest.mark.requires_mitmproxy
class TestMitmProxy(H2DownloadHandlerMixin, TestMitmProxyBase):
pass
@pytest.mark.requires_internet
class TestRealWebsite(H2DownloadHandlerMixin, TestRealWebsiteBase):
@property
def platform_cert_store_works(self) -> bool:
return sys.platform != "win32"

337
tests/test_downloader_handlers_http_base.py
View File

@ -6,7 +6,6 @@ import gzip
import json
import logging
import os
import platform
import re
import sys
from abc import ABC, abstractmethod
@ -18,6 +17,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
from urllib.parse import urlparse
import pytest
from cryptography.x509 import load_der_x509_certificate
from twisted.internet.ssl import Certificate
from twisted.python.failure import Failure
@ -33,12 +33,13 @@ from scrapy.exceptions import (
UnsupportedURLSchemeError,
)
from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse
from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE
from tests.mockserver.mitm_proxy import MitmProxy, wrong_credentials
from tests.mockserver.mitm_proxy import wrong_credentials
from tests.mockserver.proxy_echo import ProxyEchoMockServer
from tests.mockserver.simple_https import SimpleMockServer
from tests.spiders import (
@ -72,6 +73,7 @@ class TestHttpBase(ABC):
handler_supports_http2_dataloss: bool = True
# default headers added by the underlying library that cannot be suppressed
always_present_req_headers: ClassVar[frozenset[str]] = frozenset()
default_handler_settings: ClassVar[dict[str, Any]] = {}
@property
@abstractmethod
@ -82,6 +84,10 @@ class TestHttpBase(ABC):
async def get_dh(
self, settings_dict: dict[str, Any] | None = None
) -> AsyncGenerator[DownloadHandlerProtocol]:
settings_dict = {
**self.default_handler_settings,
**(settings_dict or {}),
}
crawler = get_crawler(DefaultSpider, settings_dict)
crawler.spider = crawler._create_spider()
dh = build_from_crawler(self.download_handler_cls, crawler)
@ -338,9 +344,7 @@ class TestHttpBase(ABC):
@coroutine_test
async def test_timeout_download_from_spider_server_hangs(
self,
mockserver: MockServer,
reactor_pytest: str,
self, mockserver: MockServer, reactor_pytest: str
) -> None:
if reactor_pytest == "asyncio" and sys.platform == "win32":
# https://twistedmatrix.com/trac/ticket/10279
@ -524,7 +528,7 @@ class TestHttpBase(ABC):
await download_handler.download_request(request)
assert "download_latency" in request.meta
latency = request.meta["download_latency"]
if sys.version_info < (3, 13) and platform.system() == "Windows":
if sys.version_info < (3, 13) and sys.platform == "win32":
# time.monotonic() resolution is too low here:
# https://docs.python.org/3/whatsnew/3.13.html#time
assert latency >= 0
@ -803,6 +807,25 @@ class TestHttpBase(ABC):
"The 'bindaddress' request meta key is not supported by" in caplog.text
)
@coroutine_test
async def test_verbatim_url(self, mockserver: MockServer) -> None:
# Square brackets are encoded by safe_url_string (w3lib).
path = "/uri/items?data[0]=a"
url = mockserver.url(path, is_secure=self.is_secure)
# Without verbatim_url, the brackets are percent-encoded before the
# request reaches the server.
request = Request(url)
async with self.get_dh() as download_handler:
response = await download_handler.download_request(request)
assert response.body == b"/uri/items?data%5B0%5D=a"
# With verbatim_url=True the URL is sent to the server as-is.
request = Request(url, meta={"verbatim_url": True})
async with self.get_dh() as download_handler:
response = await download_handler.download_request(request)
assert response.body == path.encode()
class TestHttpsBase(TestHttpBase):
is_secure = True
@ -875,7 +898,7 @@ class TestSimpleHttpsBase(ABC):
@pytest.fixture(scope="class")
def simple_mockserver(self) -> Generator[SimpleMockServer]:
with SimpleMockServer(
self.keyfile, self.certfile, self.cipher_string
self.keyfile, self.certfile, cipher_string=self.cipher_string
) as simple_mockserver:
yield simple_mockserver
@ -938,6 +961,143 @@ class TestHttpsCustomCiphersBase(TestSimpleHttpsBase):
cipher_string = "CAMELLIA256-SHA"
class TestHttpsTLSVersionBase(ABC):
keyfile = "keys/localhost.key"
certfile = "keys/localhost.crt"
@property
@abstractmethod
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
raise NotImplementedError
@asynccontextmanager
async def get_dh(
self, client_tls_min: str | None, client_tls_max: str | None
) -> AsyncGenerator[DownloadHandlerProtocol]:
settings = {}
if client_tls_min is not None:
settings["DOWNLOAD_TLS_MIN_VERSION"] = client_tls_min
if client_tls_max is not None:
settings["DOWNLOAD_TLS_MAX_VERSION"] = client_tls_max
crawler = get_crawler(DefaultSpider, settings_dict=settings)
crawler.spider = crawler._create_spider()
dh = build_from_crawler(self.download_handler_cls, crawler)
try:
yield dh
finally:
await dh.close()
@pytest.mark.parametrize(
(
"server_tls_min",
"server_tls_max",
"client_tls_min",
"client_tls_max",
"expect_success",
),
[
pytest.param(None, None, None, None, True, id="no-limits"),
pytest.param(None, None, None, "TLSv1.2", True, id="client-max-tls1.2"),
pytest.param(None, None, "TLSv1.3", None, True, id="client-min-tls1.3"),
pytest.param(
"TLSv1.3",
None,
None,
"TLSv1.2",
False,
id="client-max-below-server-min",
),
pytest.param(
None,
"TLSv1.2",
"TLSv1.3",
None,
False,
id="client-min-above-server-max",
),
pytest.param(None, "TLSv1.2", None, "TLSv1.2", True, id="both-tls1.2"),
pytest.param("TLSv1.3", None, "TLSv1.3", None, True, id="both-tls1.3"),
pytest.param(
None,
None,
"TLSv1.0",
None,
True,
id="client-min-tls1.0",
marks=pytest.mark.filterwarnings(
r"ignore:ssl\.TLSVersion\.TLSv1 is deprecated:DeprecationWarning"
),
),
pytest.param(
"TLSv1.0",
None,
"TLSv1.0",
None,
True,
id="both-min-tls1.0",
marks=pytest.mark.filterwarnings(
r"ignore:ssl\.TLSVersion\.TLSv1 is deprecated:DeprecationWarning"
),
),
pytest.param(
"TLSv1.2",
"TLSv1.3",
"TLSv1.2",
"TLSv1.3",
True,
id="both-tls1.2-1.3",
marks=pytest.mark.xfail(
TWISTED_TLS_LIMITS_OFFBY1,
reason="Can't set max to 1.3 on this Twisted version",
strict=True,
),
),
],
)
@coroutine_test
async def test_download(
self,
server_tls_min: str | None,
server_tls_max: str | None,
client_tls_min: str | None,
client_tls_max: str | None,
expect_success: bool,
) -> None:
with SimpleMockServer(
self.keyfile,
self.certfile,
tls_min_version=server_tls_min,
tls_max_version=server_tls_max,
) as simple_mockserver:
url = f"https://localhost:{simple_mockserver.port(is_secure=True)}/file"
request = Request(url)
async with self.get_dh(client_tls_min, client_tls_max) as dh:
if expect_success:
response = await dh.download_request(request)
assert response.body == b"0123456789"
else:
with pytest.raises(
(DownloadConnectionRefusedError, DownloadFailedError)
):
await dh.download_request(request)
@coroutine_test
async def test_invalid_min_version_setting(self) -> None:
with pytest.raises(
ValueError, match="Unknown DOWNLOAD_TLS_MIN_VERSION value: invalid"
):
async with self.get_dh(client_tls_min="invalid", client_tls_max=None):
pass
@coroutine_test
async def test_invalid_max_version_setting(self) -> None:
with pytest.raises(
ValueError, match="Unknown DOWNLOAD_TLS_MAX_VERSION value: invalid"
):
async with self.get_dh(client_tls_min=None, client_tls_max="invalid"):
pass
class TestHttpWithCrawlerBase(ABC):
@property
@abstractmethod
@ -982,15 +1142,19 @@ class TestHttpWithCrawlerBase(ABC):
if not self.is_secure:
pytest.skip("Only applies to HTTPS")
# copy of TestCrawl.test_response_ssl_certificate()
# the current test implementation can only work for Twisted-based download handlers
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
url = mockserver.url("/echo?body=test", is_secure=self.is_secure)
await crawler.crawl_async(seed=url, mockserver=mockserver)
assert isinstance(crawler.spider, SingleRequestSpider)
cert = crawler.spider.meta["responses"][0].certificate
assert isinstance(cert, Certificate)
assert cert.getSubject().commonName == b"localhost"
assert cert.getIssuer().commonName == b"localhost"
assert cert is not None
if isinstance(cert, Certificate): # Twisted
assert cert.getSubject().commonName == b"localhost"
assert cert.getIssuer().commonName == b"localhost"
elif isinstance(cert, bytes): # DER bytes
cert_x509 = load_der_x509_certificate(cert)
assert cert_x509.subject.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE"
assert cert_x509.issuer.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE"
@coroutine_test
async def test_response_ip_address(self, mockserver: MockServer) -> None:
@ -1156,6 +1320,7 @@ class TestHttpProxyBase(ABC):
class TestMitmProxyBase(ABC):
# whether the handler supports HTTPS proxies with HTTPS destinations
handler_supports_tls_in_tls: bool = True
handler_supports_socks: bool = False
@property
@abstractmethod
@ -1165,13 +1330,10 @@ class TestMitmProxyBase(ABC):
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@pytest.mark.usefixtures("mitm_proxy_server")
@coroutine_test
async def test_http_proxy(
self,
caplog: pytest.LogCaptureFixture,
mockserver: MockServer,
mitm_proxy_server: MitmProxy,
https_dest: bool,
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
) -> None:
"""HTTP proxy, HTTP or HTTPS destination."""
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
@ -1186,13 +1348,10 @@ class TestMitmProxyBase(ABC):
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@pytest.mark.usefixtures("mitm_proxy_server_https")
@coroutine_test
async def test_https_proxy(
self,
caplog: pytest.LogCaptureFixture,
mockserver: MockServer,
mitm_proxy_server_https: MitmProxy,
https_dest: bool,
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
) -> None:
"""HTTPS proxy, HTTP or HTTPS destination."""
if https_dest and not self.handler_supports_tls_in_tls:
@ -1209,13 +1368,13 @@ class TestMitmProxyBase(ABC):
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@pytest.mark.usefixtures("mitm_proxy_server")
@coroutine_test
async def test_http_proxy_auth_error(
self,
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
mockserver: MockServer,
mitm_proxy_server: MitmProxy,
https_dest: bool,
) -> None:
"""HTTP proxy, HTTP or HTTPS destination, wrong proxy creds."""
@ -1233,13 +1392,10 @@ class TestMitmProxyBase(ABC):
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@pytest.mark.usefixtures("mitm_proxy_server")
@coroutine_test
async def test_dont_leak_proxy_authorization_header(
self,
caplog: pytest.LogCaptureFixture,
mockserver: MockServer,
mitm_proxy_server: MitmProxy,
https_dest: bool,
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
) -> None:
"""HTTP proxy, HTTP or HTTPS destination. Check that the auth header
is not sent to the destination."""
@ -1253,6 +1409,49 @@ class TestMitmProxyBase(ABC):
echo = json.loads(crawler.spider.meta["responses"][0].text)
assert "Proxy-Authorization" not in echo["headers"]
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@pytest.mark.usefixtures("socks5_proxy_server")
@coroutine_test
async def test_download_with_socks_proxy(
self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool
) -> None:
"""SOCKS5 proxy, HTTP or HTTPS destination."""
if not self.handler_supports_socks:
pytest.skip("SOCKS proxies are not supported")
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async(
seed=mockserver.url("/status?n=200", is_secure=https_dest)
)
assert isinstance(crawler.spider, SingleRequestSpider)
self._assert_got_response_code(200, caplog.text)
self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
)
@pytest.mark.usefixtures("socks5_proxy_server")
@coroutine_test
async def test_socks_proxy_auth_error(
self,
caplog: pytest.LogCaptureFixture,
monkeypatch: pytest.MonkeyPatch,
mockserver: MockServer,
https_dest: bool,
) -> None:
if not self.handler_supports_socks:
pytest.skip("SOCKS proxies are not supported")
envvar = "https_proxy" if https_dest else "http_proxy"
monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar]))
crawler = get_crawler(SimpleSpider, self.settings_dict)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async(
mockserver.url("/status?n=200", is_secure=https_dest)
)
assert "DownloadConnectionRefusedError" in caplog.text
@staticmethod
def _assert_headers(headers: Headers, https_dest: bool) -> None:
assert b"X-Via-Mitmproxy" in headers
@ -1266,3 +1465,87 @@ class TestMitmProxyBase(ABC):
@staticmethod
def _assert_got_auth_exception(log: str) -> None:
assert "Proxy Authentication Required" in log or "407" in log
class TestRealWebsiteBase(ABC):
@property
@abstractmethod
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
raise NotImplementedError
@property
@abstractmethod
def settings_dict(self) -> dict[str, Any] | None:
raise NotImplementedError
@property
def platform_cert_store_works(self) -> bool:
"""Whether valid certificates can be verified.
Twisted on Windows cannot do that out of the box, see e.g.
https://github.com/twisted/twisted/issues/6371.
"""
return True
@asynccontextmanager
async def get_dh(
self, settings_dict: dict[str, Any] | None = None
) -> AsyncGenerator[DownloadHandlerProtocol]:
crawler = get_crawler(DefaultSpider, settings_dict)
crawler.spider = crawler._create_spider()
dh = build_from_crawler(self.download_handler_cls, crawler)
try:
yield dh
finally:
await dh.close()
@coroutine_test
async def test_download(self) -> None:
request = Request("https://books.toscrape.com/")
async with self.get_dh() as download_handler:
response = await download_handler.download_request(request)
assert response.status == 200
assert "All products | Books to Scrape - Sandbox" in response.text
@coroutine_test
async def test_download_with_spider(self) -> None:
crawler = get_crawler(SingleRequestSpider, self.settings_dict)
await maybe_deferred_to_future(
crawler.crawl(seed=Request("https://books.toscrape.com/"))
)
assert isinstance(crawler.spider, SingleRequestSpider)
failure = crawler.spider.meta.get("failure")
assert failure is None
reason = crawler.spider.meta["close_reason"]
assert reason == "finished"
@coroutine_test
async def test_verify_certs(self) -> None:
if not self.platform_cert_store_works:
pytest.skip("Cannot verify certificates")
request = Request("https://books.toscrape.com/")
async with self.get_dh(
{"DOWNLOAD_VERIFY_CERTIFICATES": True}
) as download_handler:
response = await download_handler.download_request(request)
assert response.status == 200
assert "All products | Books to Scrape - Sandbox" in response.text
@pytest.mark.parametrize("verify_certs", [True, False])
@coroutine_test
async def test_tls_logging(
self, caplog: pytest.LogCaptureFixture, verify_certs: bool
) -> None:
if verify_certs and not self.platform_cert_store_works:
pytest.skip("Cannot verify certificates")
request = Request("https://books.toscrape.com/")
async with self.get_dh(
{
"DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True,
"DOWNLOAD_VERIFY_CERTIFICATES": verify_certs,
}
) as download_handler:
with caplog.at_level("DEBUG"):
response = await download_handler.download_request(request)
assert response.status == 200
assert "SSL connection to books.toscrape.com using protocol" in caplog.text

2
tests/test_downloadermiddleware_cookies.py
View File

@ -42,7 +42,7 @@ def _cookie_to_set_cookie_value(cookie):
def _cookies_to_set_cookie_list(cookies):
"""Given a group of cookie defined either as a dictionary or as a list of
dictionaries (i.e. in a format supported by the cookies parameter of
Request), return the equivalen list of strings that can be associated to a
Request), return the equivalent list of strings that can be associated to a
``Set-Cookie`` header."""
if not cookies:
return []

114
tests/test_downloadermiddleware_retry.py
View File

@ -84,6 +84,39 @@ class TestRetry:
)
assert self.crawler.stats.get_value("retry/count") == 2
def test_give_up_log_level_setting(self):
crawler = get_crawler(
DefaultSpider, settings_dict={"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"}
)
crawler.spider = crawler._create_spider()
mw = RetryMiddleware.from_crawler(crawler)
mw.max_retry_times = 0
req = Request("http://example.com/503")
rsp = Response("http://example.com/503", body=b"", status=503)
with LogCapture() as log:
assert mw.process_response(req, rsp) is rsp
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"WARNING",
f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable",
)
)
def test_give_up_log_level_meta(self):
self.mw.max_retry_times = 0
req = Request("http://example.com/503", meta={"give_up_log_level": "WARNING"})
rsp = Response("http://example.com/503", body=b"", status=503)
with LogCapture() as log:
assert self.mw.process_response(req, rsp) is rsp
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"WARNING",
f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable",
)
)
def test_twistederrors(self):
exceptions = [
ConnectError,
@ -612,6 +645,87 @@ class TestGetRetryRequest:
)
)
def test_give_up_log_level_default(self):
request = Request("https://example.com")
spider = self.get_spider()
with LogCapture() as log:
get_retry_request(
request,
spider=spider,
max_retry_times=0,
)
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"ERROR",
f"Gave up retrying {request} (failed 1 times): unspecified",
)
)
def test_give_up_log_level_argument_name(self):
request = Request("https://example.com")
spider = self.get_spider()
with LogCapture() as log:
get_retry_request(
request,
spider=spider,
max_retry_times=0,
give_up_log_level="WARNING",
)
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"WARNING",
f"Gave up retrying {request} (failed 1 times): unspecified",
)
)
def test_give_up_log_level_argument_number(self):
request = Request("https://example.com")
spider = self.get_spider()
with LogCapture() as log:
get_retry_request(
request,
spider=spider,
max_retry_times=0,
give_up_log_level=logging.WARNING,
)
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"WARNING",
f"Gave up retrying {request} (failed 1 times): unspecified",
)
)
def test_give_up_log_level_setting(self):
request = Request("https://example.com")
spider = self.get_spider({"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"})
with LogCapture() as log:
get_retry_request(
request,
spider=spider,
max_retry_times=0,
)
log.check_present(
(
"scrapy.downloadermiddlewares.retry",
"WARNING",
f"Gave up retrying {request} (failed 1 times): unspecified",
)
)
def test_give_up_log_level_invalid(self):
request = Request("https://example.com")
spider = self.get_spider()
with pytest.raises(ValueError, match="Invalid give-up log level"):
get_retry_request(
request,
spider=spider,
max_retry_times=0,
give_up_log_level="NOT_A_LEVEL",
)
def test_custom_stats_key(self):
request = Request("https://example.com")
spider = self.get_spider()

2
tests/test_engine_loop.py
View File

@ -263,7 +263,7 @@ class TestRequestSendOrder:
@coroutine_test
async def test_shared_queues(self):
"""If SCHEDULER_START_*_QUEUE is falsy, start requests and other
requests share the same queue, i.e. start requests are not priorized
requests share the same queue, i.e. start requests are not prioritized
over other requests if their priority matches."""
nums = list(range(1, 14))
response_seconds = 0

6
tests/test_feedexport.py
View File

@ -41,7 +41,7 @@ from tests.spiders import ItemSpider
from tests.utils.decorators import coroutine_test, inline_callbacks_test
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
from collections.abc import Awaitable, Callable, Iterable
def path_to_url(path: str | Path) -> str:
@ -1312,7 +1312,9 @@ class TestFeedExporterSignals:
self.feed_slot_closed_received = True
async def run_signaled_feed_exporter(
self, feed_exporter_signal_handler: Callable, feed_slot_signal_handler: Callable
self,
feed_exporter_signal_handler: Callable[[], Awaitable[None] | None],
feed_slot_signal_handler: Callable[[Any], Awaitable[None] | None],
) -> None:
crawler = get_crawler(settings_dict=self.settings)
feed_exporter = FeedExporter.from_crawler(crawler)

2
tests/test_feedexport_batch.py
View File

@ -29,7 +29,7 @@ if TYPE_CHECKING:
from os import PathLike
def build_url(path: str | PathLike) -> str:
def build_url(path: str | PathLike[str]) -> str:
path_str = str(path)
if path_str[0] != "/":
path_str = "/" + path_str

28
tests/test_feedexport_storages.py
View File

@ -513,6 +513,34 @@ class TestGCSFeedStorage:
client_mock.get_bucket.assert_called_once_with("mybucket")
bucket_mock.blob.assert_called_once_with("export.csv")
blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
f.close.assert_called_once_with()
@coroutine_test
async def test_store_closes_file_on_upload_error(self):
try:
from google.cloud.storage import Client # noqa: F401,PLC0415
except ImportError:
pytest.skip("GCSFeedStorage requires google-cloud-storage")
uri = "gs://mybucket/export.csv"
project_id = "myproject-123"
acl = "publicRead"
(client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
blob_mock.upload_from_file.side_effect = OSError("Upload failed")
with mock.patch("google.cloud.storage.Client") as m:
m.return_value = client_mock
f = mock.Mock()
storage = GCSFeedStorage(uri, project_id, acl)
with pytest.raises(OSError, match="Upload failed"):
await maybe_deferred_to_future(storage.store(f))
f.seek.assert_called_once_with(0)
m.assert_called_once_with(project=project_id)
client_mock.get_bucket.assert_called_once_with("mybucket")
bucket_mock.blob.assert_called_once_with("export.csv")
blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
f.close.assert_called_once_with()
def test_overwrite_default(self):
with LogCapture() as log:

2
tests/test_http2_client_protocol.py
View File

@ -61,7 +61,7 @@ def make_html_body(val: str) -> bytes:
class DummySpider(Spider):
name = "dummy"
start_urls: list = []
start_urls = []
def parse(self, response):
print(response)

13
tests/test_http_request.py
View File

@ -162,6 +162,19 @@ class TestRequest:
r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
assert r4.url == "http://www.example.org/r%E9sum%E9.html"
def test_url_verbatim(self):
r = self.request_class(
url="http://www.scrapy.org/price/£",
meta={"verbatim_url": True},
)
assert r.url == "http://www.scrapy.org/price/£"
r = self.request_class(
url="http://www.scrapy.org/blank space",
meta={"verbatim_url": True},
)
assert r.url == "http://www.scrapy.org/blank space"
def test_body(self):
r1 = self.request_class(url="http://www.example.com/")
assert r1.body == b""

2
tests/test_loader.py
View File

@ -39,7 +39,7 @@ class AttrsNameItem:
@dataclasses.dataclass
class NameDataClass:
name: list = dataclasses.field(default_factory=list)
name: list[str] = dataclasses.field(default_factory=list)
# test item loaders

8
tests/test_pipeline_files.py
View File

@ -383,11 +383,11 @@ class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin):
class FilesPipelineTestDataClass:
name: str
# default fields
file_urls: list = dataclasses.field(default_factory=list)
files: list = dataclasses.field(default_factory=list)
file_urls: list[str] = dataclasses.field(default_factory=list)
files: list[dict[str, str]] = dataclasses.field(default_factory=list)
# overridden fields
custom_file_urls: list = dataclasses.field(default_factory=list)
custom_files: list = dataclasses.field(default_factory=list)
custom_file_urls: list[str] = dataclasses.field(default_factory=list)
custom_files: list[dict[str, str]] = dataclasses.field(default_factory=list)
class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin):

8
tests/test_pipeline_images.py
View File

@ -314,11 +314,11 @@ class TestImagesPipelineFieldsItem(TestImagesPipelineFieldsMixin):
class ImagesPipelineTestDataClass:
name: str
# default fields
image_urls: list = dataclasses.field(default_factory=list)
images: list = dataclasses.field(default_factory=list)
image_urls: list[str] = dataclasses.field(default_factory=list)
images: list[dict[str, str]] = dataclasses.field(default_factory=list)
# overridden fields
custom_image_urls: list = dataclasses.field(default_factory=list)
custom_images: list = dataclasses.field(default_factory=list)
custom_image_urls: list[str] = dataclasses.field(default_factory=list)
custom_images: list[dict[str, str]] = dataclasses.field(default_factory=list)
class TestImagesPipelineFieldsDataClass(TestImagesPipelineFieldsMixin):

4
tests/test_request_cb_kwargs.py
View File

@ -38,8 +38,8 @@ class InjectArgumentsSpiderMiddleware:
if request.callback.__name__ == "parse_spider_mw":
request.cb_kwargs["from_process_spider_input"] = True
def process_spider_output(self, response, result):
for element in result:
async def process_spider_output(self, response, result):
async for element in result:
if (
isinstance(element, Request)
and element.callback.__name__ == "parse_spider_mw_2"

4
tests/test_scheduler_base.py
View File

@ -41,10 +41,10 @@ class MinimalScheduler:
class SimpleScheduler(MinimalScheduler):
def open(self, spider: Spider) -> defer.Deferred:
def open(self, spider: Spider) -> defer.Deferred[str]:
return defer.succeed("open")
def close(self, reason: str) -> defer.Deferred:
def close(self, reason: str) -> defer.Deferred[str]:
return defer.succeed("close")
def __len__(self) -> int:

Some files were not shown because too many files have changed in this diff Show More