diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 49ea3277a..ed2388a59 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -22,10 +22,10 @@ jobs: TOXENV: pylint - python-version: "3.10" env: - TOXENV: typing + TOXENV: mypy - python-version: "3.10" env: - TOXENV: typing-tests + TOXENV: mypy-tests # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version. - python-version: "3.14" env: diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index b72a16f67..a57ea3797 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -45,26 +45,26 @@ jobs: env: TOXENV: pypy3 - # pinned deps + # min deps - python-version: "3.10.19" env: - TOXENV: pinned + TOXENV: min - python-version: "3.10.19" env: - TOXENV: default-reactor-pinned + TOXENV: min-default-reactor - python-version: "3.10.19" env: - TOXENV: no-reactor-pinned + TOXENV: min-no-reactor # pinned due to https://github.com/pypy/pypy/issues/5388 - python-version: pypy3.11-7.3.20 env: - TOXENV: pypy3-pinned + TOXENV: min-pypy3 - python-version: "3.10.19" env: - TOXENV: extra-deps-pinned + TOXENV: min-extra-deps - python-version: "3.10.19" env: - TOXENV: botocore-pinned + TOXENV: min-botocore - python-version: "3.14" env: @@ -103,7 +103,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') || contains(matrix.python-version, '3.15') + if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'min') || contains(matrix.python-version, '3.15') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 840c7f68e..f413782bc 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -41,13 +41,13 @@ jobs: env: TOXENV: no-reactor - # pinned deps + # min deps - python-version: "3.10.11" env: - TOXENV: pinned + TOXENV: min - python-version: "3.10.11" env: - TOXENV: extra-deps-pinned + TOXENV: min-extra-deps - python-version: "3.14" env: diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..24a426d36 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,6 @@ +cff-version: 1.2.0 +message: If you use Scrapy in published research, please cite it as below. +title: Scrapy +authors: + - name: Scrapy contributors +url: https://scrapy.org diff --git a/SECURITY.md b/SECURITY.md index 752d6318a..49cafdf9e 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,8 +4,8 @@ | Version | Supported | | ------- | ------------------ | -| 2.15.x | :white_check_mark: | -| < 2.15.x | :x: | +| 2.16.x | :white_check_mark: | +| < 2.16.x | :x: | ## Reporting a Vulnerability diff --git a/conftest.py b/conftest.py index fcbf59426..cf0568111 100644 --- a/conftest.py +++ b/conftest.py @@ -24,13 +24,13 @@ def _py_files(folder): collect_ignore = [ # may need extra deps "docs/_ext", - # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerProcessSubprocess *_py_files("tests/AsyncCrawlerProcess"), - # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerRunnerSubprocess *_py_files("tests/AsyncCrawlerRunner"), - # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), - # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerRunnerSubprocess *_py_files("tests/CrawlerRunner"), ] @@ -99,6 +99,19 @@ def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmPr proxy.stop() +@pytest.fixture # function scope because it modifies os.environ +def socks5_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]: + proxy = MitmProxy(mode="socks5") + url = proxy.start() + monkeypatch.setenv("http_proxy", url) + monkeypatch.setenv("https_proxy", url) + + try: + yield proxy + finally: + proxy.stop() + + @pytest.fixture(scope="session") def reactor_pytest(request) -> str: return request.config.getoption("--reactor") diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 6ec565e24..29394799b 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -1,6 +1,6 @@ {% extends "!layout.html" %} -{# Overriden to include a link to scrapy.org, not just to the docs root #} +{# Overridden to include a link to scrapy.org, not just to the docs root #} {%- block sidebartitle %} {# the logo helper function was removed in Sphinx 6 and deprecated since Sphinx 4 #} diff --git a/docs/faq.rst b/docs/faq.rst index 87adbfa4b..df90122f5 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -82,10 +82,18 @@ to steal from us! Does Scrapy work with HTTP proxies? ----------------------------------- -Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP -Proxy downloader middleware. See +Yes. Support for HTTP proxies is provided through the HTTP Proxy downloader +middleware. See :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`. +Does Scrapy work with SOCKS proxies? +------------------------------------ + +Yes, when using +:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. See +:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and the +handler documentation. + How can I scrape an item with attributes in different pages? ------------------------------------------------------------ @@ -360,7 +368,10 @@ method for this purpose. For example: Does Scrapy support IPv6 addresses? ----------------------------------- -Yes, by setting :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. +Yes, but when using +:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` or +:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` you need to +set :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. Note that by doing so, you lose the ability to set a specific timeout for DNS requests (the value of the :setting:`DNS_TIMEOUT` setting is ignored). diff --git a/docs/news.rst b/docs/news.rst index 7f3952eaf..b8b976df9 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,17 +3,284 @@ Release notes ============= -Scrapy VERSION (unreleased) ---------------------------- +.. _release-2.16.0: + +Scrapy 2.16.0 (2026-05-19) +-------------------------- + +Highlights: + +- Official support for Python 3.14 + +- Support for Twisted 26.4.0+ + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Increased the minimum versions of the following dependencies: + + - service_identity_: 18.1.0 → 23.1.0 + + (:issue:`7347`) + +- Added support for Twisted 26.4.0+. + (:issue:`7347`, :issue:`7505`, :issue:`7520`) + +- Added support for Python 3.14. + (:issue:`6604`, :issue:`7460`) Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` has been made - private as it's an implementation detail of - :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. +- The following classes and functions, intended for internal use by + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + and :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`, have + been made private: + + - ``scrapy.core.downloader.handlers.http11.ScrapyAgent`` + + - ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingTCP4ClientEndpoint`` + + - ``scrapy.core.downloader.handlers.http11.tunnel_request_data()`` + + - ``scrapy.core.downloader.handlers.http2.ScrapyH2Agent`` + + (:issue:`7496`, :issue:`7510`) + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.FormRequest`` is deprecated. You can use the :doc:`form2request + ` library instead, see :ref:`form`. + (:issue:`6438`) + +- ``scrapy.utils.python.MutableChain`` is deprecated. + (:issue:`7504`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The ``start_requests()`` method of :class:`~scrapy.Spider`, deprecated in + 2.13.0, is removed and no longer called. Use :meth:`~scrapy.Spider.start` + instead, or both to maintain support for lower Scrapy versions. + (:issue:`7490`) + +- Support for ``process_start_requests()`` methods of :ref:`spider middlewares + `, deprecated in 2.13.0, is removed. Use + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, + or both to maintain support for lower Scrapy versions. + (:issue:`7490`) + +- Support for synchronous ``process_spider_output()`` methods of spider + middlewares, deprecated in Scrapy 2.13.0, is removed. You should upgrade + the affected middlewares to have asynchronous ``process_spider_output()`` + methods. + (:issue:`7504`) + +- The ``spider`` arguments of the following methods of + :class:`~scrapy.core.scraper.Scraper`, deprecated in Scrapy 2.13.0, are + removed: + + - ``close_spider()`` + + - ``enqueue_scrape()`` + + - ``handle_spider_error()`` + + - ``handle_spider_output()`` + + (:issue:`7487`) + +- HTTP/1.0 support code, deprecated in Scrapy 2.13.0, is removed. This + includes: + + - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler`` + + - The ``scrapy.core.downloader.webclient`` module. + + - The ``DOWNLOADER_HTTPCLIENTFACTORY`` setting. + + (:issue:`7486`) + +- The following functions, deprecated in Scrapy 2.13.0, are removed, you + should import them from :mod:`w3lib.url` directly instead: + + - ``scrapy.utils.url.add_or_replace_parameter()`` + + - ``scrapy.utils.url.add_or_replace_parameters()`` + + - ``scrapy.utils.url.any_to_uri()`` + + - ``scrapy.utils.url.canonicalize_url()`` + + - ``scrapy.utils.url.file_uri_to_path()`` + + - ``scrapy.utils.url.is_url()`` + + - ``scrapy.utils.url.parse_data_uri()`` + + - ``scrapy.utils.url.parse_url()`` + + - ``scrapy.utils.url.path_to_file_uri()`` + + - ``scrapy.utils.url.safe_download_url()`` + + - ``scrapy.utils.url.safe_url_string()`` + + - ``scrapy.utils.url.url_query_cleaner()`` + + - ``scrapy.utils.url.url_query_parameter()`` + + (:issue:`7487`) + +- The following test-related code, deprecated in Scrapy 2.13.0, is removed: + + - the ``scrapy.utils.testproc`` module + + - the ``scrapy.utils.testsite`` module + + - ``scrapy.utils.test.assert_gcs_environ()`` + + - ``scrapy.utils.test.get_ftp_content_and_delete()`` + + - ``scrapy.utils.test.get_gcs_content_and_delete()`` + + - ``scrapy.utils.test.mock_google_cloud_storage()`` + + - ``scrapy.utils.test.skip_if_no_boto()`` + + - ``scrapy.utils.test.TestSpider`` + + (:issue:`7487`) + +- ``scrapy.utils.versions.scrapy_components_versions()``, deprecated in + Scrapy 2.13.0, is removed, you can use + :func:`scrapy.utils.versions.get_versions` instead. + (:issue:`7487`) + +- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` and + ``scrapy.utils.url.escape_ajax()``, deprecated in Scrapy 2.13.0, are + removed. + (:issue:`7487`) + +- The ``__init__()`` method of priority queue classes (see + :setting:`SCHEDULER_PRIORITY_QUEUE`) now needs to support a keyword-only + ``start_queue_cls`` parameter, not supporting it was deprecated in Scrapy + 2.13.0. + (:issue:`7487`) + +- ``scrapy.spiders.init.InitSpider``, deprecated in Scrapy 2.13.0, is + removed. + (:issue:`7487`) + +New features +~~~~~~~~~~~~ + +- New features and improvements for + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`: + + - Support for proxies. + + - Support for the :reqmeta:`download_latency` meta key. + + - Support for :attr:`Response.certificate + `. + + - Default headers set by the ``httpx`` library are no longer added to + requests. + + (:issue:`7441`, :issue:`7524`) + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now + skips HTTPS proxy certificate verification when the + :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting is set to ``False``. (:issue:`7496`) +Improvements +~~~~~~~~~~~~ + +- :func:`time.monotonic` is used instead of :func:`time.time` to calculate + elapsed time in various places. + (:issue:`7377`) + +- Improved extraction of the file extension from the URL in + :class:`~scrapy.pipelines.files.FilesPipeline`. + (:issue:`4225`, :issue:`7414`) + +- Other code refactoring and improvements. + (:issue:`7401`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now + raises an exception when a request has an ``https://`` destination and an + ``https://`` proxy, which is not supported by this handler. Previously it + tried to connect to the proxy via HTTP in this case. + (:issue:`7496`) + +- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now + raises an exception for requests with ``http://`` URLs instead of trying to + connect, which is not supported by this handler. + (:issue:`7496`) + +- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` no longer + adds the ``:status`` pseudo-header to :attr:`Response.headers + `. + (:issue:`7441`) + +- Fixed :func:`scrapy.utils.response.open_in_browser` removing the ```` + tag when adding the ```` tag. + (:issue:`7459`) + +Documentation +~~~~~~~~~~~~~ + +- Documented that + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + doesn't support HTTPS proxies for HTTPS destinations and that + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` doesn't + support proxies at all. + (:issue:`7496`) + +- Added an example of using + :class:`logging.handlers.TimedRotatingFileHandler` to rotate Scrapy logs. + (:issue:`3628`, :issue:`7501`) + +- Added a ``CITATION.cff`` file. + (:issue:`7502`, :issue:`7519`) + +- Mentioned ``DOWNLOADER_CLIENT_TLS_METHOD`` in :ref:`bans`. + (:issue:`5232`, :issue:`7518`) + +- Other documentation improvements and fixes. + (:issue:`7417`, + :issue:`7463`, + :issue:`7472`, + :issue:`7480`, + :issue:`7489`, + :issue:`7503`, + :issue:`7507`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added tests that connect to https://books.toscrape.com/ to test the + behavior with a real website. These tests are marked with the + ``requires_internet`` pytest mark and can be skipped with e.g. + ``-m 'not requires_internet'`` if you cannot or don't want to run them. + (:issue:`7520`) + +- Type hints improvements and fixes. + (:issue:`7492`, :issue:`7532`) + +- CI and test improvements and fixes. + (:issue:`7441`, :issue:`7466`, :issue:`7491`, :issue:`7496`) + .. _release-2.15.2: Scrapy 2.15.2 (2026-04-28) @@ -1366,8 +1633,8 @@ Highlights: - Added the :reqmeta:`allow_offsite` request meta key -- :ref:`Spider middlewares that don't support asynchronous spider output - ` are deprecated +- Spider middlewares that don't support asynchronous spider output are + deprecated - Added a base class for :ref:`universal spider middlewares ` @@ -1505,13 +1772,11 @@ Deprecations ``start_queue_cls`` parameter. (:issue:`6752`) -- :ref:`Spider middlewares that don't support asynchronous spider output - ` are deprecated. The async iterable - downgrading feature, needed for using such middlewares with asynchronous - callbacks and with other spider middlewares that produce asynchronous - iterables, is also deprecated. Please update all such middlewares to - support asynchronous spider output. - (:issue:`6664`) +- Spider middlewares that don't support asynchronous spider output are + deprecated. The async iterable downgrading feature, needed for using such + middlewares with asynchronous callbacks and with other spider middlewares + that produce asynchronous iterables, is also deprecated. Please update all + such middlewares to support asynchronous spider output. (:issue:`6664`) - Functions that were imported from :mod:`w3lib.url` and re-exported in :mod:`scrapy.utils.url` are now deprecated, you should import them from @@ -1770,9 +2035,8 @@ Documentation - Documented the setting values set in the default project template. (:issue:`6762`, :issue:`6775`) -- Improved the :ref:`docs ` about asynchronous - iterable support in spider middlewares. - (:issue:`6688`) +- Improved the docs about asynchronous iterable support in spider + middlewares. (:issue:`6688`) - Improved the :ref:`docs ` about using :class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based @@ -1936,7 +2200,7 @@ Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - User-defined cookies for HTTPS requests will have the ``secure`` flag set - to ``True`` unless it's set to ``False`` explictly. This is important when + to ``True`` unless it's set to ``False`` explicitly. This is important when these cookies are reused in HTTP requests, e.g. after a redirect to an HTTP URL. (:issue:`6357`) @@ -1971,7 +2235,7 @@ Backward-incompatible changes ``crawler.settings`` instead. When they call ``__init__()`` of the base class they should pass the ``crawler`` argument to it too. - A ``from_settings()`` method shouldn't be defined. Class-specific - initialization code should go into either an overriden ``from_crawler()`` + initialization code should go into either an overridden ``from_crawler()`` method or into ``__init__()``. - It's now possible to override ``from_crawler()`` and it's not necessary to call ``MediaPipeline.from_crawler()`` in it if other recommendations @@ -7200,7 +7464,7 @@ This 1.1 release brings a lot of interesting features and bug fixes: selectors engine without needing to upgrade Scrapy. - HTTPS downloader now does TLS protocol negotiation by default, instead of forcing TLS 1.0. You can also set the SSL/TLS method - using the new :setting:`DOWNLOADER_CLIENT_TLS_METHOD`. + using the new ``DOWNLOADER_CLIENT_TLS_METHOD`` setting. - These bug fixes may require your attention: diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index ea3a60b4c..ba68f0dbc 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -23,9 +23,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :class:`~scrapy.Request` callbacks. - If you are using any custom or third-party :ref:`spider middleware - `, see :ref:`sync-async-spider-middleware`. - - The :meth:`process_item` method of :ref:`item pipelines `. @@ -39,13 +36,9 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` - method of :ref:`spider middlewares `. - - If defined as a coroutine, it must be an :term:`asynchronous generator`. - The input ``result`` parameter is an :term:`asynchronous iterable`. - - See also :ref:`sync-async-spider-middleware` and - :ref:`universal-spider-middleware`. + method of :ref:`spider middlewares `, which + *must* be defined as an :term:`asynchronous generator` except in + :ref:`universal spider middlewares `. - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method of :ref:`spider middlewares `, which *must* be @@ -277,136 +270,3 @@ You can also send multiple requests in parallel: "price": responses[0][1].css(".price::text").get(), "price2": responses[1][1].css(".color::text").get(), } - - -.. _sync-async-spider-middleware: - -Mixing synchronous and asynchronous spider middlewares -====================================================== - -The output of a :class:`~scrapy.Request` callback is passed as the ``result`` -parameter to the -:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method -of the first :ref:`spider middleware ` from the -:ref:`list of active spider middlewares `. -Then the output of that ``process_spider_output`` method is passed to the -``process_spider_output`` method of the next spider middleware, and so on for -every active spider middleware. - -Scrapy supports mixing :ref:`coroutine methods ` and synchronous methods -in this chain of calls. - -However, if any of the ``process_spider_output`` methods is defined as a -synchronous method, and the previous ``Request`` callback or -``process_spider_output`` method is a coroutine, there are some drawbacks to -the asynchronous-to-synchronous conversion that Scrapy does so that the -synchronous ``process_spider_output`` method gets a synchronous iterable as its -``result`` parameter: - -- The whole output of the previous ``Request`` callback or - ``process_spider_output`` method is awaited at this point. - -- If an exception raises while awaiting the output of the previous - ``Request`` callback or ``process_spider_output`` method, none of that - output will be processed. - - This contrasts with the regular behavior, where all items yielded before - an exception raises are processed. - -Asynchronous-to-synchronous conversions are supported for backward -compatibility, but they are deprecated and will stop working in a future -version of Scrapy. - -To avoid asynchronous-to-synchronous conversions, when defining ``Request`` -callbacks as coroutine methods or when using spider middlewares whose -``process_spider_output`` method is an :term:`asynchronous generator`, all -active spider middlewares must either have their ``process_spider_output`` -method defined as an asynchronous generator or :ref:`define a -process_spider_output_async method `. - -.. _sync-async-spider-middleware-users: - -For middleware users --------------------- - -If you have asynchronous callbacks or use asynchronous-only spider middlewares -you should make sure the asynchronous-to-synchronous conversions -:ref:`described above ` don't happen. To do this, -make sure all spider middlewares you use support asynchronous spider output. -Even if you don't have asynchronous callbacks and don't use asynchronous-only -spider middlewares in your project, it's still a good idea to make sure all -middlewares you use support asynchronous spider output, so that it will be easy -to start using asynchronous callbacks in the future. Because of this, Scrapy -logs a warning when it detects a synchronous-only spider middleware. - -If you want to update middlewares you wrote, see the :ref:`following section -`. If you have 3rd-party middlewares that -aren't yet updated by their authors, you can :ref:`subclass ` -them to make them :ref:`universal ` and use the -subclasses in your projects. - -.. _sync-async-spider-middleware-authors: - -For middleware authors ----------------------- - -If you have a spider middleware that defines a synchronous -``process_spider_output`` method, you should update it to support asynchronous -spider output for :ref:`better compatibility `, -even if you don't yet use it with asynchronous callbacks, especially if you -publish this middleware for other people to use. You have two options for this: - -1. Make the middleware asynchronous, by making the ``process_spider_output`` - method an :term:`asynchronous generator`. -2. Make the middleware universal, as described in the :ref:`next section - `. - -If your middleware won't be used in projects with synchronous-only middlewares, -e.g. because it's an internal middleware and you know that all other -middlewares in your projects are already updated, it's safe to choose the first -option. Otherwise, it's better to choose the second option. - -.. _universal-spider-middleware: - -Universal spider middlewares ----------------------------- - -To allow writing a spider middleware that supports asynchronous execution of -its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding -:ref:`asynchronous-to-synchronous conversions `) -while maintaining support for older Scrapy versions, you may define -``process_spider_output`` as a synchronous method and define an -:term:`asynchronous generator` version of that method with an alternative name: -``process_spider_output_async``. - -For example: - -.. code-block:: python - - class UniversalSpiderMiddleware: - def process_spider_output(self, response, result): - for r in result: - # ... do something with r - yield r - - async def process_spider_output_async(self, response, result): - async for r in result: - # ... do something with r - yield r - -.. note:: This is an interim measure to allow, for a time, to write code that - works in Scrapy 2.7 and later without requiring - asynchronous-to-synchronous conversions, and works in earlier Scrapy - versions as well. - - In some future version of Scrapy, however, this feature will be - deprecated and, eventually, in a later version of Scrapy, this - feature will be removed, and all spider middlewares will be expected - to define their ``process_spider_output`` method as an asynchronous - generator. - -Since 2.13.0, Scrapy provides a base class, -:class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which implements -the ``process_spider_output()`` and ``process_spider_output_async()`` methods, -so instead of duplicating the processing code you can override the -``get_processed_request()`` and/or the ``get_processed_item()`` method. diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 1e95864c6..888bfaf08 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -130,44 +130,39 @@ these exceptions. .. _download-handlers-ref: -Built-in download handlers reference -==================================== +Built-in HTTP download handlers reference +========================================= -DataURIDownloadHandler ----------------------- +Scrapy ships several handlers for HTTP and HTTPS requests. While all of them +support basic features, they may differ in support of specific Scrapy features +and settings and HTTP protocol features. See the documentation of specific +handlers and specific settings for more information. Additionally, as the +underlying HTTP client implementations differ between handlers, the behavior of +specific websites may be different when doing the same Scrapy requests but +using different handlers. -.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler +Here is a comparison of some features of the built-in HTTP handlers, see the +individual handler docs for more differences: -| Supported scheme: ``data``. -| Lazy: no. +================== ================= ===================== ==================== +Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler +================== ================= ===================== ==================== +Requires asyncio No No Yes +Requires a reactor Yes Yes No +HTTP/1.1 No Yes Yes +HTTP/2 Yes No Yes +TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl`` +HTTP proxies No Yes Yes +SOCKS proxies No No Yes +================== ================= ===================== ==================== -This handler supports RFC 2397 ``data:content/type;base64,`` data URIs. +You can find additional HTTP download handlers in the +scrapy-download-handlers-incubator_ package. This package is made by the Scrapy +developers and contains experimental handlers that may be included in some +later Scrapy version but can already be used. Please refer to the documentation +of this package for more information. -FileDownloadHandler -------------------- - -.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler - -| Supported scheme: ``file``. -| Lazy: no. - -This handler supports ``file:///path`` local file URIs. It doesn't -support remote files. - -FTPDownloadHandler ------------------- - -.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler - -| Supported scheme: ``ftp``. -| Lazy: no. - -This handler supports ``ftp://host/path`` FTP URIs. - -It's implemented using :mod:`twisted.protocols.ftp`. - -.. note:: - This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. +.. _scrapy-download-handlers-incubator: https://github.com/scrapy-plugins/scrapy-download-handlers-incubator .. _twisted-http2-handler: @@ -177,7 +172,9 @@ H2DownloadHandler .. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler | Supported scheme: ``https``. -| Lazy: yes. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocol for them. @@ -196,63 +193,96 @@ If you want to use this handler you need to replace the default one for the "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler", } +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + .. warning:: This handler is experimental, and not yet recommended for production environments. Future Scrapy versions may introduce related changes without a deprecation period or warning. -.. note:: +=========================== ================================================ +HTTP proxies No (not implemented) +SOCKS proxies No (not supported by the library) +HTTP/2 Yes +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ - Known limitations of the HTTP/2 implementation in this handler include: +Other limitations: - - No support for proxies. +- No support for HTTP/1.1. - - No support for HTTP/2 Cleartext (h2c), since no major browser supports - HTTP/2 unencrypted (refer `http2 faq`_). +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. - - No setting to specify a maximum `frame size`_ larger than the default - value, 16384. Connections to servers that send a larger frame will - fail. +- No support for the :signal:`bytes_received` and :signal:`headers_received` + signals. - - No support for `server pushes`_, which are ignored. +Known limitations of the HTTP/2 support: - - No support for the :signal:`bytes_received` and - :signal:`headers_received` signals. +- No support for HTTP/2 Cleartext (h2c), since no major browser supports + HTTP/2 unencrypted (refer `http2 faq`_). + +- No setting to specify a maximum `frame size`_ larger than the default + value, 16384. Connections to servers that send a larger frame will fail. + +- No support for `server pushes`_, which are ignored. .. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2 .. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption .. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2 -.. note:: - This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. - HTTP11DownloadHandler --------------------- .. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler | Supported schemes: ``http``, ``https``. -| Lazy: no. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. This handler supports ``http://host/path`` and ``https://host/path`` URLs and uses the HTTP/1.1 protocol for them. It's implemented using :mod:`twisted.web.client`. -.. note:: - This handler is not supported when :setting:`TWISTED_REACTOR_ENABLED` is ``False``. +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +=========================== ================================================ +HTTP proxies Yes +SOCKS proxies No (not supported by the library) +HTTP/2 No (implemented as a separate handler) +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ + +Other limitations: + +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. + +- HTTPS proxies to HTTPS destinations are not supported. HttpxDownloadHandler -------------------- +.. versionadded:: 2.15.0 + .. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler | Supported schemes: ``http``, ``https``. -| Lazy: no. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: yes. +| :ref:`Requires a Twisted reactor `: no. This handler supports ``http://host/path`` and ``https://host/path`` URLs and -uses the HTTP/1.1 protocol for them. +uses the HTTP/1.1 or HTTP/2 protocol for them. It's implemented using the ``httpx`` library and needs it to be installed. @@ -266,33 +296,83 @@ If you want to use this handler you need to replace the default ones for the "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", } +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + .. warning:: This handler is experimental, and not yet recommended for production environments. Future Scrapy versions may introduce related changes without a deprecation period or warning or even remove it altogether. -.. note:: +=========================== ======================================= +HTTP proxies Yes +SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``) +HTTP/2 Yes (requires ``httpx[http2]``) +``response.certificate`` DER bytes +Per-request ``bindaddress`` No (not supported by the library) +TLS implementation Standard library ``ssl`` +=========================== ======================================= - As this handler is based on a different HTTP client implementation compared - to :class:`~.HTTP11DownloadHandler`, it's expected that its behavior on - some websites may be different. Additionally, these are the Scrapy features - that are explicitly not supported when using it: +Other limitations: - - Proxy support (the :reqmeta:`proxy` meta key). +- The handler creates a separate connection pool for each proxy URL (due to + limitations of ``httpx``) which may lead to higher resource usage when + using proxy rotation. - - Per-request bind address support (the :reqmeta:`bindaddress` meta key). - The global :setting:`DOWNLOAD_BIND_ADDRESS` setting is supported but the - port number, if specified, will be ignored. +.. setting:: HTTPX_HTTP2_ENABLED - - The :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` and - :setting:`DOWNLOADER_CLIENT_TLS_METHOD` settings. +HTTPX_HTTP2_ENABLED +^^^^^^^^^^^^^^^^^^^ - - Settings specific to the Twisted networking or HTTP implementation, like - :setting:`DNS_RESOLVER`. +Default: ``False`` - - Using :ref:`non-asyncio reactors ` (``httpx`` requires - ``asyncio``). +Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra +needs to be installed if you want to enable this setting. + +.. versionadded:: VERSION + +Built-in non-HTTP download handlers reference +============================================= + +DataURIDownloadHandler +---------------------- + +.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler + +| Supported scheme: ``data``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports RFC 2397 ``data:content/type;base64,`` data URIs. + +FileDownloadHandler +------------------- + +.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler + +| Supported scheme: ``file``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``file:///path`` local file URIs. It doesn't +support remote files. + +FTPDownloadHandler +------------------ + +.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler + +| Supported scheme: ``ftp``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``ftp://host/path`` FTP URIs. + +It's implemented using :mod:`twisted.protocols.ftp`. S3DownloadHandler ----------------- @@ -300,7 +380,9 @@ S3DownloadHandler .. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler | Supported scheme: ``s3``. -| Lazy: yes. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. This handler supports ``s3://bucket/path`` S3 URIs. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index a51e431d2..8cb29deff 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -745,8 +745,7 @@ HttpProxyMiddleware Handling of this meta key needs to be implemented inside the :ref:`download handler `, so it's not guaranteed to be supported by all 3rd-party handlers. It's currently unsupported by - :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` and - :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`. .. note:: @@ -758,6 +757,13 @@ HttpProxyMiddleware :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` supports HTTPS proxies only for HTTP destinations. +.. note:: + + If the download handler supports it, you can use a SOCKS proxy URL (e.g. + ``socks5://username:password@some_proxy_server:port``). + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` + supports SOCKS proxies while other built-in handlers don't. + HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1053,6 +1059,21 @@ has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught exception propagation, see :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`. +.. setting:: RETRY_GIVE_UP_LOG_LEVEL + +RETRY_GIVE_UP_LOG_LEVEL +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``"ERROR"`` + +:ref:`Logging level ` used for the message logged when a request +exceeds its retries. + +Can be a level name (e.g. ``"WARNING"``) or a number (e.g. ``logging.WARNING`` +or ``30``). + +See also: :reqmeta:`give_up_log_level`, :func:`get_retry_request`. + .. setting:: RETRY_PRIORITY_ADJUST RETRY_PRIORITY_ADJUST diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst index 74256eef4..c4cd05683 100644 --- a/docs/topics/exporters.rst +++ b/docs/topics/exporters.rst @@ -93,24 +93,25 @@ described next. 1. Declaring a serializer in the field -------------------------------------- -If you use :class:`~scrapy.Item` you can declare a serializer in the -:ref:`field metadata `. The serializer must be -a callable which receives a value and returns its serialized form. +Every :ref:`item type ` except :class:`dict` lets you declare a +serializer in the :ref:`field metadata `. The serializer +must be a callable which receives a value and returns its serialized form. Example: .. code-block:: python - import scrapy + from dataclasses import dataclass, field def serialize_price(value): return f"$ {str(value)}" - class Product(scrapy.Item): - name = scrapy.Field() - price = scrapy.Field(serializer=serialize_price) + @dataclass + class Product: + name: str + price: float = field(metadata={"serializer": serialize_price}) 2. Overriding the serialize_field() method diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst index f0a8bb5ed..735e46c29 100644 --- a/docs/topics/extensions.rst +++ b/docs/topics/extensions.rst @@ -136,6 +136,19 @@ Core Stats extension Enable the collection of core statistics, provided the stats collection is enabled (see :ref:`topics-stats`). +The following stats are collected: + +* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`). +* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`). +* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`). +* ``finish_reason``: the closing reason string (e.g. ``"finished"``, + ``"closespider_timeout"``). +* ``item_scraped_count``: total number of items that passed all pipelines. +* ``item_dropped_count``: total number of items dropped by a pipeline. +* ``item_dropped_reasons_count/``: per-exception drop count + (e.g. ``item_dropped_reasons_count/DropItem``). +* ``response_received_count``: total number of HTTP responses received. + .. _topics-extensions-ref-telnetconsole: Log Count extension @@ -247,6 +260,7 @@ settings: * :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM` * :setting:`CLOSESPIDER_ITEMCOUNT` * :setting:`CLOSESPIDER_PAGECOUNT` +* :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM` * :setting:`CLOSESPIDER_ERRORCOUNT` .. note:: @@ -260,12 +274,11 @@ settings: CLOSESPIDER_TIMEOUT """"""""""""""""""" -Default: ``0`` +Default: ``0.0`` -An integer which specifies a number of seconds. If the spider remains open for -more than that number of seconds, it will be automatically closed with the -reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by -timeout. +If the spider remains open for more than this number of seconds, it will be +automatically closed with the reason ``closespider_timeout``. If zero (or non +set), spiders won't be closed by timeout. .. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst index 5ad005893..3ccc90bf9 100644 --- a/docs/topics/loaders.rst +++ b/docs/topics/loaders.rst @@ -102,14 +102,13 @@ One approach to overcome this is to define items using the .. code-block:: python from dataclasses import dataclass, field - from typing import Optional @dataclass class InventoryItem: - name: Optional[str] = field(default=None) - price: Optional[float] = field(default=None) - stock: Optional[int] = field(default=None) + name: str | None = field(default=None) + price: float | None = field(default=None) + stock: int | None = field(default=None) .. _topics-loaders-processors: @@ -228,7 +227,8 @@ metadata. Here is an example: .. code-block:: python - import scrapy + from dataclasses import dataclass, field + from itemloaders.processors import Join, MapCompose, TakeFirst from w3lib.html import remove_tags @@ -238,14 +238,21 @@ metadata. Here is an example: return value - class Product(scrapy.Item): - name = scrapy.Field( - input_processor=MapCompose(remove_tags), - output_processor=Join(), + @dataclass + class Product: + name: str | None = field( + default=None, + metadata={ + "input_processor": MapCompose(remove_tags), + "output_processor": Join(), + }, ) - price = scrapy.Field( - input_processor=MapCompose(remove_tags, filter_price), - output_processor=TakeFirst(), + price: str | None = field( + default=None, + metadata={ + "input_processor": MapCompose(remove_tags, filter_price), + "output_processor": TakeFirst(), + }, ) diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index 8c04c578d..b542c6c05 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -81,9 +81,6 @@ thumbnailing and normalizing images to JPEG/RGB format. Enabling your Media Pipeline ============================ -.. setting:: IMAGES_STORE -.. setting:: FILES_STORE - To enable your media pipeline you must first add it to your project :setting:`ITEM_PIPELINES` setting. @@ -102,6 +99,8 @@ For Files Pipeline, use: .. note:: You can also use both the Files and Images Pipeline at the same time. +.. setting:: IMAGES_STORE +.. setting:: FILES_STORE Then, configure the target storage setting to a valid value that will be used for storing the downloaded images. Otherwise the pipeline will remain disabled, @@ -337,17 +336,18 @@ respectively), the pipeline will put the results under the respective field When using :ref:`item types ` for which fields are defined beforehand, you must define both the URLs field and the results field. For example, when using the images pipeline, items must define both the ``image_urls`` and the -``images`` field. For instance, using the :class:`~scrapy.Item` class: +``images`` field. For instance, using a dataclass: .. code-block:: python - import scrapy + from dataclasses import dataclass, field - class MyItem(scrapy.Item): + @dataclass + class MyItem: # ... other item fields ... - image_urls = scrapy.Field() - images = scrapy.Field() + image_urls: list[str] = field(default_factory=list) + images: list[dict] = field(default_factory=list) If you want to use another field name for the URLs key or for the results key, it is also possible to override it. diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst index 4f036db29..11c2656da 100644 --- a/docs/topics/practices.rst +++ b/docs/topics/practices.rst @@ -409,6 +409,10 @@ Here are some tips to keep in mind when dealing with these kinds of sites: * use a pool of rotating IPs. For example, the free `Tor project`_ or paid services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a super proxy that you can attach your own proxies to. +* for HTTPS websites, if blocking appears related to TLS behavior, consider + adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond + differently depending on the TLS method used by the client. * use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy plugin `__ and additional features, like `AI web scraping `__ diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index ba7b3ee3c..8fd3de621 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -117,6 +117,9 @@ Request objects :param encoding: the encoding of this request (defaults to ``'utf-8'``). This encoding will be used to percent-encode the URL and to convert the body to bytes (if given as a string). + + To disable URL percent-encoding for a request, use the + :reqmeta:`verbatim_url` request meta key. :type encoding: str :param priority: sets :attr:`priority`, defaults to ``0``. @@ -136,9 +139,13 @@ Request objects .. attribute:: Request.url - A string containing the URL of this request. Keep in mind that this - attribute contains the escaped URL, so it can differ from the URL passed in - the ``__init__()`` method. + A string containing the URL of this request. + + Keep in mind that this attribute contains the escaped URL, so it can + differ from the URL passed in the ``__init__()`` method. + + If :reqmeta:`verbatim_url` is set to ``True``, the URL is kept as + passed to ``__init__()``. This attribute is read-only. To change the URL of a Request use :meth:`replace`. @@ -541,6 +548,11 @@ in your :meth:`fingerprint` method implementation: .. autofunction:: scrapy.utils.request.fingerprint +By default, request fingerprinting canonicalizes the request URL. If +:reqmeta:`verbatim_url` is set to ``True``, fingerprinting does not +canonicalize the URL, and the ``keep_fragments`` parameter is ignored (it is +effectively true). + For example, to take the value of a request header named ``X-ID`` into account: @@ -702,6 +714,7 @@ Those are: * :reqmeta:`download_timeout` * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * ``ftp_user`` (See :setting:`FTP_USER` for more info) +* :reqmeta:`give_up_log_level` * :reqmeta:`handle_httpstatus_all` * :reqmeta:`handle_httpstatus_list` * :reqmeta:`is_start_request` @@ -710,6 +723,7 @@ Those are: * :reqmeta:`redirect_reasons` * :reqmeta:`redirect_urls` * :reqmeta:`referrer_policy` +* :reqmeta:`verbatim_url` .. reqmeta:: bindaddress @@ -777,15 +791,38 @@ download_fail_on_dataloss Whether or not to fail on broken responses. See: :setting:`DOWNLOAD_FAIL_ON_DATALOSS`. +.. reqmeta:: give_up_log_level + +give_up_log_level +----------------- + +:ref:`Logging level ` used for the message logged when a request +exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details. + .. reqmeta:: max_retry_times max_retry_times --------------- -The meta key is used set retry times per request. When initialized, the +The meta key is used set retry times per request. When set, the :reqmeta:`max_retry_times` meta key takes higher precedence over the :setting:`RETRY_TIMES` setting. +.. reqmeta:: verbatim_url + +verbatim_url +------------ + +Set this key to ``True`` to keep the request URL as passed to +:class:`~scrapy.Request`, without URL percent-encoding. + +When this key is enabled, :func:`~scrapy.utils.request.fingerprint` does not +canonicalize the request URL, so requests whose URLs differ only in +characters that would otherwise be canonicalized get different fingerprints. + +In this mode, the ``keep_fragments`` parameter is ignored, and it is +effectively true. + .. _topics-stop-response-download: @@ -917,7 +954,7 @@ Response objects :type request: scrapy.Request :param certificate: an object representing the server's SSL certificate. - :type certificate: twisted.internet.ssl.Certificate + :type certificate: typing.Any :param ip_address: The IP address of the server from which the Response originated. :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` @@ -1009,8 +1046,8 @@ Response objects .. attribute:: Response.certificate - A :class:`twisted.internet.ssl.Certificate` object representing - the server's SSL certificate. + An object representing the server's SSL certificate. Its type and + contents depend on the download handler that produced the response. Only populated for ``https`` responses, ``None`` otherwise. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index cf2658d60..06de33e6f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -737,32 +737,50 @@ specific cipher that is not included in ``DEFAULT`` if a website requires it. by all 3rd-party handlers. It's currently unsupported by :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. -.. setting:: DOWNLOADER_CLIENT_TLS_METHOD +.. setting:: DOWNLOAD_TLS_MAX_VERSION -DOWNLOADER_CLIENT_TLS_METHOD ----------------------------- +DOWNLOAD_TLS_MAX_VERSION +------------------------ -Default: ``'TLS'`` +Default: ``None`` -Use this setting to customize the TLS/SSL method used by the HTTPS download -handler. +Use this setting to change the maximum version of the TLS protocol allowed to +be used by Scrapy. -This setting must be one of these string values: +This setting must be either ``None``, in which case it doesn't affect the +version selection, or one of these string values: -- ``'TLS'``: maps to OpenSSL's ``TLS_method()`` (a.k.a ``SSLv23_method()``), - which allows protocol negotiation, starting from the highest supported - by the platform; **default, recommended** -- ``'TLSv1.0'``: this value forces HTTPS connections to use TLS version 1.0 ; - set this if you want the behavior of Scrapy<1.1 -- ``'TLSv1.1'``: forces TLS version 1.1 -- ``'TLSv1.2'``: forces TLS version 1.2 +- ``'TLSv1.0'`` +- ``'TLSv1.1'`` +- ``'TLSv1.2'`` +- ``'TLSv1.3'`` + +The range of allowed TLS versions advertised by Scrapy when making TLS +connections will depend on the TLS implementation defaults and the values of +:setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION`. +It's possible to re-enable versions that are supported by the TLS +implementation but disabled by default by adjusting these settings, but it's +impossible to enable unsupported ones, such as any versions below 1.2 in many +modern environments. .. note:: Handling of this setting needs to be implemented inside the :ref:`download handler `, so it's not guaranteed to be supported - by all 3rd-party handlers. It's currently unsupported by - :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. + by all 3rd-party handlers. Additionally, the set of supported TLS versions + depends on the TLS implementation being used by the handler. + +.. setting:: DOWNLOAD_TLS_MIN_VERSION + +DOWNLOAD_TLS_MIN_VERSION +------------------------ + +Default: ``None`` + +Use this setting to change the minimum version of the TLS protocol allowed to +be used by Scrapy. + +See :setting:`DOWNLOAD_TLS_MAX_VERSION` for the details and limitations. .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 8b000697a..99bbdf292 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -117,36 +117,28 @@ one or more of these methods: :type response: :class:`~scrapy.http.Response` object .. method:: process_spider_output(response, result) + :async: - This method is called with the results returned from the Spider, after - it has processed the response. + This method is an :term:`asynchronous generator` called with the + results from the spider after the spider has processed the response. - :meth:`process_spider_output` must return an iterable of - :class:`~scrapy.Request` objects and :ref:`item objects - `. - - Consider defining this method as an :term:`asynchronous generator`, - which will be a requirement in a future version of Scrapy. However, if - you plan on sharing your spider middleware with other people, consider - either :ref:`enforcing Scrapy 2.7 ` - as a minimum requirement of your spider middleware, or :ref:`making - your spider middleware universal ` so that - it works with Scrapy versions earlier than Scrapy 2.7. + .. seealso:: :ref:`universal-spider-middleware`. :param response: the response which generated this output from the spider :type response: :class:`~scrapy.http.Response` object - :param result: the result returned by the spider - :type result: an iterable of :class:`~scrapy.Request` objects and - :ref:`item objects ` + :param result: the results from the spider + :type result: an :term:`asynchronous iterable` of + :class:`~scrapy.Request` objects and :ref:`item objects + ` .. method:: process_spider_output_async(response, result) :async: - If defined, this method must be an :term:`asynchronous generator`, - which will be called instead of :meth:`process_spider_output` if - ``result`` is an :term:`asynchronous iterable`. + Alternative name for :meth:`process_spider_output` used when + implementing a :ref:`universal spider middleware + `. .. method:: process_spider_exception(response, exception) @@ -174,13 +166,40 @@ one or more of these methods: :type exception: :exc:`Exception` object +.. _universal-spider-middleware: + +Universal spider middlewares +---------------------------- + +In Scrapy 2.6.3 and lower, ``process_spider_output()`` must be a *synchronous* +generator. + +To support those versions and higher Scrapy versions in the same middleware, +rename your asynchronous :meth:`~SpiderMiddleware.process_spider_output` +method to :meth:`~SpiderMiddleware.process_spider_output_async`, and define a +synchronous ``process_spider_output()`` method to be used by 2.6.3 and lower +versions. + +For example: + +.. code-block:: python + + class UniversalSpiderMiddleware: + async def process_spider_output_async(self, response, result): + async for r in result: + # ... do something with r + yield r + + def process_spider_output(self, response, result): + for r in result: + # ... do something with r + yield r + Base class for custom spider middlewares ---------------------------------------- Scrapy provides a base class for custom spider middlewares. It's not required -to use it but it can help with simplifying middleware implementations and -reducing the amount of boilerplate code in :ref:`universal middlewares -`. +to use it but it can help with simplifying middleware implementations. .. module:: scrapy.spidermiddlewares.base diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index 806af509f..bcef9d5f6 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -457,13 +457,14 @@ with a ``TestItem`` declared in a ``myproject.items`` module: .. code-block:: python - import scrapy + from dataclasses import dataclass - class TestItem(scrapy.Item): - id = scrapy.Field() - name = scrapy.Field() - description = scrapy.Field() + @dataclass + class TestItem: + id: str | None = None + name: str | None = None + description: str | None = None .. currentmodule:: scrapy.spiders @@ -556,7 +557,6 @@ Let's now take a look at an example CrawlSpider with rules: .. code-block:: python - import scrapy from scrapy.spiders import CrawlSpider, Rule from scrapy.linkextractors import LinkExtractor @@ -576,7 +576,7 @@ Let's now take a look at an example CrawlSpider with rules: def parse_item(self, response): self.logger.info("Hi, this is an item page! %s", response.url) - item = scrapy.Item() + item = {} item["id"] = response.xpath('//td[@id="item_id"]/text()').re(r"ID: (\d+)") item["name"] = response.xpath('//td[@id="item_name"]/text()').get() item["description"] = response.xpath( @@ -714,9 +714,9 @@ These spiders are pretty easy to use, let's have a look at one example: ) item = TestItem() - item["id"] = node.xpath("@id").get() - item["name"] = node.xpath("name").get() - item["description"] = node.xpath("description").get() + item.id = node.xpath("@id").get() + item.name = node.xpath("name").get() + item.description = node.xpath("description").get() return item Basically what we did up there was to create a spider that downloads a feed from @@ -778,9 +778,9 @@ Let's see an example similar to the previous one, but using a self.logger.info("Hi, this is a row!: %r", row) item = TestItem() - item["id"] = row["id"] - item["name"] = row["name"] - item["description"] = row["description"] + item.id = row["id"] + item.name = row["name"] + item.description = row["description"] return item diff --git a/pyproject.toml b/pyproject.toml index 6c9ff3100..2d857dc6d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,7 +86,6 @@ pattern = "^(?P.+)$" [tool.mypy] strict = true -allow_any_generics = true # 67 errors extra_checks = false # weird addErrback() errors untyped_calls_exclude = [ "twisted", @@ -138,7 +137,7 @@ module = [ ignore_missing_imports = true [tool.bumpversion] -current_version = "2.15.2" +current_version = "2.16.0" commit = true tag = true tag_name = "{new_version}" @@ -158,6 +157,8 @@ parse = """(?P0|[1-9]\\d*)\\.(?P0|[1-9]\\d*)""" serialize = ["{major}.{minor}"] [tool.coverage.run] +# sysmon, default on 3.14, is too slow: https://github.com/coveragepy/coveragepy/issues/2172 +core = "ctrace" branch = true include = ["scrapy/*"] omit = ["tests/*"] @@ -226,6 +227,7 @@ disable = [ "too-many-positional-arguments", "too-many-public-methods", "too-many-return-statements", + "undefined-variable", "unused-argument", "unused-variable", "useless-import-alias", # used as a hint to mypy @@ -267,6 +269,7 @@ markers = [ "requires_botocore: marks tests that need botocore (but not boto3)", "requires_boto3: marks tests that need botocore and boto3", "requires_mitmproxy: marks tests that need mitmproxy", + "requires_internet: marks tests that need real Internet access", ] filterwarnings = [ "ignore::DeprecationWarning:twisted.web.static", diff --git a/scrapy/VERSION b/scrapy/VERSION index 07d875c2d..752490696 100644 --- a/scrapy/VERSION +++ b/scrapy/VERSION @@ -1 +1 @@ -2.15.2 +2.16.0 diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 17bcf19b0..cc8624fa1 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -147,7 +147,7 @@ class Command(ScrapyCommand): name: str, url: str, template_name: str, - template_file: str | os.PathLike, + template_file: str | os.PathLike[str], ) -> None: """Generate the spider module, based on the given template""" assert self.settings is not None diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 24f56b7a6..c0da7dfa4 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -27,7 +27,7 @@ class Contract: request_cls: type[Request] | None = None name: str - def __init__(self, method: Callable, *args: Any): + def __init__(self, method: Callable[..., Any], *args: Any): self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook") self.testcase_post = _create_testcase(method, f"@{self.name} post-hook") self.args: tuple[Any, ...] = args @@ -105,7 +105,7 @@ class ContractsManager: return methods - def extract_contracts(self, method: Callable) -> list[Contract]: + def extract_contracts(self, method: Callable[..., Any]) -> list[Contract]: contracts: list[Contract] = [] assert method.__doc__ is not None for line_ in method.__doc__.split("\n"): @@ -134,7 +134,9 @@ class ContractsManager: return requests - def from_method(self, method: Callable, results: TestResult) -> Request | None: + def from_method( + self, method: Callable[..., Any], results: TestResult + ) -> Request | None: contracts = self.extract_contracts(method) if contracts: request_cls = Request @@ -170,7 +172,7 @@ class ContractsManager: return None def _clean_req( - self, request: Request, method: Callable, results: TestResult + self, request: Request, method: Callable[..., Any], results: TestResult ) -> None: """stop the request from returning objects and records any errors""" @@ -195,7 +197,7 @@ class ContractsManager: request.errback = eb_wrapper -def _create_testcase(method: Callable, desc: str) -> TestCase: +def _create_testcase(method: Callable[..., Any], desc: str) -> TestCase: spider = method.__self__.name # type: ignore[attr-defined] class ContractTestCase(TestCase): diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 7f7d34c21..40f09ffb2 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,13 +1,13 @@ from __future__ import annotations import warnings -from contextlib import contextmanager from typing import TYPE_CHECKING, Any, cast from OpenSSL import SSL from twisted.internet.ssl import ( AcceptableCiphers, CertificateOptions, + TLSVersion, optionsForClientTLS, ) from twisted.web.client import BrowserLikePolicyForHTTPS @@ -16,19 +16,19 @@ from zope.interface.declarations import implementer from zope.interface.verify import verifyObject from scrapy.core.downloader.tls import ( + _TWISTED_VERSION_MAP, DEFAULT_CIPHERS, + _openssl_methods, _ScrapyClientTLSOptions, _ScrapyClientTLSOptions26, - openssl_methods, ) from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils._deps_compat import TWISTED_TLS_NEW_IMPL from scrapy.utils.deprecate import create_deprecated_class from scrapy.utils.misc import build_from_crawler, load_object +from scrapy.utils.ssl import _get_cert_options_version_kwargs, _get_tls_version_limits if TYPE_CHECKING: - from collections.abc import Generator - from twisted.internet._sslverify import ClientTLSOptions # typing.Self requires Python 3.11 @@ -38,24 +38,14 @@ if TYPE_CHECKING: from scrapy.settings import BaseSettings -@contextmanager -def _filter_method_warning() -> Generator[None]: - with warnings.catch_warnings(): - # Twisted deprecation, https://github.com/scrapy/scrapy/issues/3288 - warnings.filterwarnings( - "ignore", - message=r"Passing method to twisted\.internet\.ssl\.CertificateOptions", - category=DeprecationWarning, - ) - yield - - @implementer(IPolicyForHTTPS) class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): """Non-peer-certificate verifying HTTPS context factory. - Default OpenSSL method is ``TLS_METHOD`` (also called ``SSLv23_METHOD``) - which allows TLS protocol negotiation. + Uses :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS`, + :setting:`DOWNLOAD_TLS_MIN_VERSION` and :setting:`DOWNLOAD_TLS_MAX_VERSION` + to configure the :class:`~twisted.internet.ssl.CertificateOptions` + instance. The purpose of this custom class is to provide a ``creatorForNetloc()`` method that returns a ``_ScrapyClientTLSOptions`` instance configured based @@ -64,15 +54,19 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): def __init__( self, - method: int = SSL.SSLv23_METHOD, # noqa: S503 + method: int | None = SSL.SSLv23_METHOD, # noqa: S503 tls_verbose_logging: bool = False, tls_ciphers: str | None = None, *args: Any, verify_certificates: bool = False, + tls_min_version: TLSVersion | None = None, + tls_max_version: TLSVersion | None = None, **kwargs: Any, ): super().__init__(*args, **kwargs) # type: ignore[no-untyped-call] - self._ssl_method: int = method + self._ssl_method: int | None = method + self.tls_min_version: TLSVersion | None = tls_min_version + self.tls_max_version: TLSVersion | None = tls_max_version self.tls_verbose_logging: bool = tls_verbose_logging # unused self.tls_ciphers: AcceptableCiphers if tls_ciphers: @@ -85,7 +79,7 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): def from_crawler( cls, crawler: Crawler, - method: int = SSL.SSLv23_METHOD, # noqa: S503 + method: int | None = SSL.SSLv23_METHOD, # noqa: S503 *args: Any, **kwargs: Any, ) -> Self: @@ -93,12 +87,21 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" ) tls_ciphers: str | None = crawler.settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] + # DOWNLOADER_CLIENT_TLS_METHOD reading and handling should be also moved here + # when the deprecated load_context_factory_from_settings() is removed + tls_min_ver, tls_max_ver = _get_tls_version_limits( + crawler.settings, _TWISTED_VERSION_MAP.__getitem__ + ) + if tls_min_ver or tls_max_ver: + method = None verify_certificates = crawler.settings.getbool("DOWNLOAD_VERIFY_CERTIFICATES") return cls( # type: ignore[misc] *args, method=method, tls_verbose_logging=tls_verbose_logging, tls_ciphers=tls_ciphers, + tls_min_version=tls_min_ver, + tls_max_version=tls_max_ver, verify_certificates=verify_certificates, **kwargs, ) @@ -108,12 +111,23 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): return self._get_cert_options() def _get_cert_options(self) -> CertificateOptions: - with _filter_method_warning(): - return _ScrapyCertificateOptions( - method=self._ssl_method, - fixBrokenPeers=True, - acceptableCiphers=self.tls_ciphers, + return _ScrapyCertificateOptions(**self._get_cert_options_kwargs()) + + def _get_cert_options_kwargs(self) -> dict[str, Any]: + kwargs: dict[str, Any] = { + "fixBrokenPeers": True, + "acceptableCiphers": self.tls_ciphers, + } + if self.tls_min_version or self.tls_max_version: + kwargs.update( + _get_cert_options_version_kwargs( + self.tls_min_version, self.tls_max_version + ) ) + # when ScrapyClientContextFactory is removed self._ssl_method can just be None by default + elif self._ssl_method != SSL.SSLv23_METHOD: + kwargs["method"] = self._ssl_method + return kwargs # should be removed together with ScrapyClientContextFactory def getContext( @@ -131,22 +145,16 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): return _ScrapyClientTLSOptions26( self._get_cert_options()._makeTLSConnection, hostname.decode("ascii"), - verbose_logging=self.tls_verbose_logging, ) return _ScrapyClientTLSOptions( hostname.decode("ascii"), # type: ignore[arg-type] self._get_context(), # type: ignore[arg-type] ) # Otherwise use the normal Twisted function. - # Note that this doesn't use self._get_context(). - with _filter_method_warning(): - return optionsForClientTLS( # type: ignore[no-any-return] - hostname=hostname.decode("ascii"), - extraCertificateOptions={ - "method": self._ssl_method, - "acceptableCiphers": self.tls_ciphers, - }, - ) + return optionsForClientTLS( # type: ignore[no-any-return] + hostname=hostname.decode("ascii"), + extraCertificateOptions=self._get_cert_options_kwargs(), + ) ScrapyClientContextFactory = create_deprecated_class( @@ -171,12 +179,6 @@ class BrowserLikeContextFactory(_ScrapyClientContextFactory): :meth:`creatorForNetloc` is the same as :class:`~twisted.web.client.BrowserLikePolicyForHTTPS` except this context factory allows setting the TLS/SSL method to use. - - The default OpenSSL method is ``TLS_METHOD`` (also called - ``SSLv23_METHOD``) which allows TLS protocol negotiation. - - As this overrides the parent ``creatorForNetloc()`` method, only - ``self._ssl_method`` is used from the parent class. """ def __init__(self, *args: Any, **kwargs: Any): @@ -190,11 +192,10 @@ class BrowserLikeContextFactory(_ScrapyClientContextFactory): super().__init__(*args, **kwargs) def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions: - with _filter_method_warning(): - return optionsForClientTLS( # type: ignore[no-any-return] - hostname=hostname.decode("ascii"), - extraCertificateOptions={"method": self._ssl_method}, - ) + return optionsForClientTLS( # type: ignore[no-any-return] + hostname=hostname.decode("ascii"), + extraCertificateOptions=self._get_cert_options_kwargs(), + ) @implementer(IPolicyForHTTPS) @@ -269,6 +270,16 @@ def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS: Also passes values of other relevant settings to the factory class. """ + tls_method_setting: str = crawler.settings["DOWNLOADER_CLIENT_TLS_METHOD"] + if tls_method_setting != "TLS": + warnings.warn( + "Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is" + " deprecated, please use DOWNLOAD_TLS_MIN_VERSION and/or" + " DOWNLOAD_TLS_MAX_VERSION instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + tls_method = _openssl_methods[tls_method_setting] if crawler.settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] == "SENTINEL": context_factory_cls = _ScrapyClientContextFactory else: # pragma: no cover @@ -280,13 +291,12 @@ def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS: context_factory_cls = load_object( crawler.settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] ) - ssl_method = openssl_methods[crawler.settings.get("DOWNLOADER_CLIENT_TLS_METHOD")] return cast( "IPolicyForHTTPS", build_from_crawler( context_factory_cls, crawler, - method=ssl_method, + method=tls_method, ), ) diff --git a/scrapy/core/downloader/handlers/_base_http.py b/scrapy/core/downloader/handlers/_base_http.py new file mode 100644 index 000000000..83d130462 --- /dev/null +++ b/scrapy/core/downloader/handlers/_base_http.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from abc import ABC +from typing import TYPE_CHECKING + +from .base import BaseDownloadHandler + +if TYPE_CHECKING: + from scrapy.crawler import Crawler + + +class BaseHttpDownloadHandler(BaseDownloadHandler, ABC): + """Base class for built-in HTTP download handlers.""" + + def __init__(self, crawler: Crawler): + super().__init__(crawler) + self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE") + self._default_warnsize: int = crawler.settings.getint("DOWNLOAD_WARNSIZE") + self._fail_on_dataloss: bool = crawler.settings.getbool( + "DOWNLOAD_FAIL_ON_DATALOSS" + ) + self._tls_verbose_logging: bool = crawler.settings.getbool( + "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" + ) + self._fail_on_dataloss_warned: bool = False diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py new file mode 100644 index 000000000..16b655716 --- /dev/null +++ b/scrapy/core/downloader/handlers/_base_streaming.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +import base64 +import logging +import time +from abc import ABC, abstractmethod +from io import BytesIO +from typing import TYPE_CHECKING, Any, ClassVar, Generic, NoReturn, TypedDict, TypeVar +from urllib.parse import quote, urlsplit + +from scrapy import Request, signals +from scrapy.exceptions import ( + DownloadCancelledError, + NotConfigured, + ResponseDataLossError, +) +from scrapy.utils._download_handlers import ( + check_stop_download, + get_dataloss_msg, + get_maxsize_msg, + get_warnsize_msg, + make_response, + normalize_bind_address, +) +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.url import add_http_if_no_scheme + +from ._base_http import BaseHttpDownloadHandler + +if TYPE_CHECKING: + from collections.abc import AsyncIterable + from contextlib import AbstractAsyncContextManager + from ipaddress import IPv4Address, IPv6Address + + from _typeshed import SizedBuffer + + # typing.NotRequired requires Python 3.11 + from typing_extensions import NotRequired + + from scrapy.crawler import Crawler + from scrapy.http import Headers, Response + + +logger = logging.getLogger(__name__) + +_ResponseT = TypeVar("_ResponseT") + + +class _BaseResponseArgs(TypedDict): + status: int + url: str + headers: Headers + certificate: NotRequired[Any] + ip_address: NotRequired[IPv4Address | IPv6Address | None] + protocol: str | None + + +class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_ResponseT]): + """A base class for HTTP download handlers that follow the streaming logic flow.""" + + _DEFAULT_CONNECT_TIMEOUT: ClassVar[float] = 10 + experimental: ClassVar[bool] = False + requires_asyncio: ClassVar[bool] = True + # require subclasses to disable proxies explicitly with an explanation + supports_proxies: ClassVar[bool] = True + supports_per_request_bindaddress: ClassVar[bool] = False + + def __init__(self, crawler: Crawler): + if self.requires_asyncio and not is_asyncio_available(): # pragma: no cover + raise NotConfigured( + f"{type(self).__name__} requires the asyncio support. Make" + f" sure that you have either enabled the asyncio Twisted" + f" reactor in the TWISTED_REACTOR setting or disabled the" + f" TWISTED_REACTOR_ENABLED setting. See the asyncio documentation" + f" of Scrapy for more information." + ) + self._check_deps_installed() + super().__init__(crawler) + if self.experimental: + logger.warning( + f"{type(self).__name__} is experimental and is not recommended for production use." + ) + self._bind_address = normalize_bind_address( + crawler.settings.get("DOWNLOAD_BIND_ADDRESS") + ) + self._proxy_auth_encoding: str = crawler.settings.get("HTTPPROXY_AUTH_ENCODING") + # these are useful for many handlers but used in different ways by them + self._pool_size_total: int = crawler.settings.getint("CONCURRENT_REQUESTS") + self._pool_size_per_host: int = crawler.settings.getint( + "CONCURRENT_REQUESTS_PER_DOMAIN" + ) + + @staticmethod + @abstractmethod + def _check_deps_installed() -> None: + """Raise NotConfigured if the required deps are not installed.""" + raise NotImplementedError + + @abstractmethod + def _make_request( + self, request: Request, timeout: float + ) -> AbstractAsyncContextManager[_ResponseT]: + """Return an async context manager yielding the library-specific response. + + Exceptions raised by the library should be reraised as Scrapy-specific ones. + """ + raise NotImplementedError + + @staticmethod + @abstractmethod + def _extract_headers(response: _ResponseT) -> Headers: + """Convert library-specific response headers to a + :class:`~scrapy.http.headers.Headers` object.""" + raise NotImplementedError + + @staticmethod + @abstractmethod + def _build_base_response_args( + response: _ResponseT, request: Request, headers: Headers + ) -> _BaseResponseArgs: + """Build kwargs for :func:`scrapy.utils._download_handlers.make_response`.""" + raise NotImplementedError + + @staticmethod + @abstractmethod + def _iter_body_chunks(response: _ResponseT) -> AsyncIterable[SizedBuffer]: + """Return an async iterable yielding body chunks from the response.""" + raise NotImplementedError + + @staticmethod + @abstractmethod + def _is_dataloss_exception(exc: Exception) -> bool: + """Return True if ``exc`` represents dataloss.""" + raise NotImplementedError + + def _log_tls_info(self, response: _ResponseT, request: Request) -> None: + """Log TLS connection details, if possible.""" + + async def download_request(self, request: Request) -> Response: + if not self.supports_proxies and request.meta.get("proxy"): + raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.") + if not self.supports_per_request_bindaddress and request.meta.get( + "bindaddress" + ): + logger.error( + f"The 'bindaddress' request meta key is not supported by" + f" {type(self).__name__} and will be ignored." + ) + timeout: float = request.meta.get( + "download_timeout", self._DEFAULT_CONNECT_TIMEOUT + ) + start_time = time.monotonic() + async with self._make_request(request, timeout) as response: + request.meta["download_latency"] = time.monotonic() - start_time + return await self._read_response(response, request) + + async def _read_response(self, response: _ResponseT, request: Request) -> Response: + maxsize: int = request.meta.get("download_maxsize", self._default_maxsize) + warnsize: int = request.meta.get("download_warnsize", self._default_warnsize) + + headers = self._extract_headers(response) + content_length = headers.get("Content-Length") + expected_size = int(content_length) if content_length is not None else None + if maxsize and expected_size and expected_size > maxsize: + self._cancel_maxsize(expected_size, maxsize, request, expected=True) + + reached_warnsize = False + if warnsize and expected_size and expected_size > warnsize: + reached_warnsize = True + logger.warning( + get_warnsize_msg(expected_size, warnsize, request, expected=True) + ) + + make_response_base_args = self._build_base_response_args( + response, request, headers + ) + + if self._tls_verbose_logging: + self._log_tls_info(response, request) + + if stop_download := check_stop_download( + signals.headers_received, + self.crawler, + request, + headers=headers, + body_length=expected_size, + ): + return make_response( + **make_response_base_args, + stop_download=stop_download, + ) + + response_body = BytesIO() + bytes_received = 0 + try: + async for chunk in self._iter_body_chunks(response): + response_body.write(chunk) + bytes_received += len(chunk) + + if stop_download := check_stop_download( + signals.bytes_received, self.crawler, request, data=chunk + ): + return make_response( + **make_response_base_args, + body=response_body.getvalue(), + stop_download=stop_download, + ) + + if maxsize and bytes_received > maxsize: + response_body.truncate(0) + self._cancel_maxsize( + bytes_received, maxsize, request, expected=False + ) + + if warnsize and bytes_received > warnsize and not reached_warnsize: + reached_warnsize = True + logger.warning( + get_warnsize_msg( + bytes_received, warnsize, request, expected=False + ) + ) + except Exception as e: + if not self._is_dataloss_exception(e): + raise + fail_on_dataloss: bool = request.meta.get( + "download_fail_on_dataloss", self._fail_on_dataloss + ) + if not fail_on_dataloss: + return make_response( + **make_response_base_args, + body=response_body.getvalue(), + flags=["dataloss"], + ) + if not self._fail_on_dataloss_warned: + logger.warning(get_dataloss_msg(request.url)) + self._fail_on_dataloss_warned = True + raise ResponseDataLossError(str(e)) from e + + return make_response( + **make_response_base_args, + body=response_body.getvalue(), + ) + + def _get_bind_address_host(self) -> str | None: + """Return the host portion of the bind address. + + Needed for handlers that don't support the bind port. + """ + if self._bind_address is None: + return None + host, port = self._bind_address + if port != 0: + logger.warning( + "DOWNLOAD_BIND_ADDRESS specifies a port (%s), but %s does not " + "support binding to a specific local port. Ignoring the port " + "and binding only to %r.", + port, + type(self).__name__, + host, + ) + return host + + @staticmethod + def _cancel_maxsize( + size: int, limit: int, request: Request, *, expected: bool + ) -> NoReturn: + warning_msg = get_maxsize_msg(size, limit, request, expected=expected) + logger.warning(warning_msg) + raise DownloadCancelledError(warning_msg) + + @staticmethod + def _extract_proxy(request: Request) -> tuple[str | None, str | None]: + """Return a tuple of the proxy URL with a scheme and the value of the + Proxy-Authorization header. + + This is useful for handlers that take the proxy headers separately. + """ + proxy: str | None = request.meta.get("proxy") + if not proxy: + return None, None + proxy = add_http_if_no_scheme(proxy) + auth_header: list[bytes] | None = request.headers.pop( + b"Proxy-Authorization", None + ) + return proxy, auth_header[0].decode("ascii") if auth_header else None + + def _extract_proxy_url_with_creds(self, request: Request) -> str | None: + """Return the proxy URL with the userinfo added based on the + Proxy-Authorization header. + + This is useful for handlers that cannot take the proxy headers + separately. + """ + proxy_url, auth_header = self._extract_proxy(request) + if proxy_url is None or auth_header is None: + return proxy_url + scheme, token = auth_header.split(" ", 1) + if scheme != "Basic": + raise ValueError( + f"Expected Basic auth in Proxy-Authorization, got {scheme}" + ) + user, password = ( + base64.b64decode(token).decode(self._proxy_auth_encoding).split(":", 1) + ) + parts = urlsplit(proxy_url) + netloc = f"{quote(user)}:{quote(password)}@{parts.netloc}" + return parts._replace(netloc=netloc).geturl() diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py index 313fb1c4f..c960fc152 100644 --- a/scrapy/core/downloader/handlers/_httpx.py +++ b/scrapy/core/downloader/handlers/_httpx.py @@ -3,136 +3,165 @@ from __future__ import annotations import ipaddress -import logging import ssl -import time -from http.cookiejar import Cookie, CookieJar -from io import BytesIO -from typing import TYPE_CHECKING, Any, NoReturn, TypedDict +from contextlib import asynccontextmanager +from socket import gaierror +from typing import TYPE_CHECKING, ClassVar -from scrapy import Request, signals from scrapy.exceptions import ( CannotResolveHostError, - DownloadCancelledError, DownloadConnectionRefusedError, DownloadFailedError, DownloadTimeoutError, NotConfigured, - ResponseDataLossError, UnsupportedURLSchemeError, ) -from scrapy.http import Headers, Response -from scrapy.utils._download_handlers import ( - BaseHttpDownloadHandler, - check_stop_download, - get_dataloss_msg, - get_maxsize_msg, - get_warnsize_msg, - make_response, - normalize_bind_address, +from scrapy.http import Headers +from scrapy.utils._download_handlers import NullCookieJar +from scrapy.utils.python import _iter_exc_causes +from scrapy.utils.ssl import ( + _log_sslobj_debug_info, + _make_insecure_ssl_ctx, + _make_ssl_context, ) -from scrapy.utils.asyncio import is_asyncio_available -from scrapy.utils.ssl import _log_sslobj_debug_info, _make_ssl_context + +from ._base_streaming import BaseStreamingDownloadHandler, _BaseResponseArgs if TYPE_CHECKING: - from contextlib import AbstractAsyncContextManager - from http.client import HTTPResponse - from ipaddress import IPv4Address, IPv6Address - from urllib.request import Request as ULRequest + from collections.abc import AsyncIterator from httpcore import AsyncNetworkStream + from scrapy import Request from scrapy.crawler import Crawler +HAS_SOCKS = HAS_HTTP2 = False + try: import httpx except ImportError: httpx = None # type: ignore[assignment] +else: + # a small hack to avoid importing these optional extras unconditionally -logger = logging.getLogger(__name__) + DOWNLOAD_FAILED_EXCEPTIONS: tuple[type[BaseException], ...] = ( + httpx.RequestError, + httpx.InvalidURL, + ) + try: + import h2.exceptions -class _BaseResponseArgs(TypedDict): - status: int - url: str - headers: Headers - ip_address: IPv4Address | IPv6Address - protocol: str - - -# workaround for (and from) https://github.com/encode/httpx/issues/2992 -class _NullCookieJar(CookieJar): # pragma: no cover - """A CookieJar that rejects all cookies.""" - - def extract_cookies(self, response: HTTPResponse, request: ULRequest) -> None: + HAS_HTTP2 = True + DOWNLOAD_FAILED_EXCEPTIONS += (h2.exceptions.InvalidBodyLengthError,) + except ImportError: # pragma: no cover pass - def set_cookie(self, cookie: Cookie) -> None: + try: + import socksio.exceptions + + HAS_SOCKS = True + DOWNLOAD_FAILED_EXCEPTIONS += (socksio.exceptions.ProtocolError,) + except ImportError: # pragma: no cover pass -class HttpxDownloadHandler(BaseHttpDownloadHandler): - _DEFAULT_CONNECT_TIMEOUT = 10 +if TYPE_CHECKING: + _Base = BaseStreamingDownloadHandler[httpx.Response] +else: + _Base = BaseStreamingDownloadHandler + + +class HttpxDownloadHandler(_Base): + experimental: ClassVar[bool] = True def __init__(self, crawler: Crawler): - # we skip HttpxDownloadHandler tests with the non-asyncio reactor - if not is_asyncio_available(): # pragma: no cover + super().__init__(crawler) + self._verify_certificates: bool = crawler.settings.getbool( + "DOWNLOAD_VERIFY_CERTIFICATES" + ) + self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED") + if self._enable_h2 and not HAS_HTTP2: # pragma: no cover raise NotConfigured( - f"{type(self).__name__} requires the asyncio support. Make" - f" sure that you have either enabled the asyncio Twisted" - f" reactor in the TWISTED_REACTOR setting or disabled the" - f" TWISTED_REACTOR_ENABLED setting. See the asyncio" - f" documentation of Scrapy for more information." + f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed." ) + self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings) + self._bind_host: str | None = self._get_bind_address_host() + self._limits: httpx.Limits = httpx.Limits( + # hard limit on simultaneous connections + max_connections=self._pool_size_total, + # total number of idle connections in the pool (extra ones are closed) + max_keepalive_connections=self._pool_size_total, + ) + + self._default_client: httpx.AsyncClient = self._make_client() + # httpx doesn't support per-request proxies: https://github.com/encode/httpx/discussions/3183, + # so we keep a pool of clients per proxy URL. LRU eviction can be added here if needed. + self._proxy_clients: dict[str, httpx.AsyncClient] = {} + + @staticmethod + def _check_deps_installed() -> None: if httpx is None: # pragma: no cover raise NotConfigured( - f"{type(self).__name__} requires the httpx library to be installed." + "HttpxDownloadHandler requires the httpx library to be installed." ) - super().__init__(crawler) - logger.warning( - "HttpxDownloadHandler is experimental and is not recommended for production use." - ) - bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS") - bind_address = normalize_bind_address(bind_address) - self._bind_address: str | None = None + def _make_client(self, proxy_url: str | None = None) -> httpx.AsyncClient: + if proxy_url: + if proxy_url.startswith("https:") and not self._verify_certificates: + proxy_ssl_context = _make_insecure_ssl_ctx() + else: + proxy_ssl_context = None + proxy = httpx.Proxy(proxy_url, ssl_context=proxy_ssl_context) + else: + proxy = None - if bind_address is not None: - host, port = bind_address - if port != 0: - logger.warning( - "DOWNLOAD_BIND_ADDRESS specifies a port (%s), but %s does not " - "support binding to a specific local port. Ignoring the port " - "and binding only to %r.", - port, - type(self).__name__, - host, - ) - self._bind_address = host - - self._client = httpx.AsyncClient( - cookies=_NullCookieJar(), + client = httpx.AsyncClient( + cookies=NullCookieJar(), transport=httpx.AsyncHTTPTransport( - verify=_make_ssl_context(crawler.settings), - local_address=self._bind_address, + verify=self._ssl_context, + local_address=self._bind_host, + http2=self._enable_h2, + limits=self._limits, + trust_env=False, + proxy=proxy, ), ) # https://github.com/encode/httpx/discussions/1566 for header_name in ("accept", "accept-encoding", "user-agent"): - self._client.headers.pop(header_name, None) + client.headers.pop(header_name, None) + return client - async def download_request(self, request: Request) -> Response: - self._warn_unsupported_meta(request.meta) + def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient: + if proxy_url is None: + return self._default_client + if cached := self._proxy_clients.get(proxy_url): + return cached + client = self._make_client(proxy_url) + self._proxy_clients[proxy_url] = client + return client + + @asynccontextmanager + async def _make_request( + self, request: Request, timeout: float + ) -> AsyncIterator[httpx.Response]: + proxy = self._extract_proxy_url_with_creds(request) + if proxy and proxy.startswith("socks") and not HAS_SOCKS: # pragma: no cover + raise ValueError( + f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed." + ) + client = self._get_client(proxy) - timeout: float = request.meta.get( - "download_timeout", self._DEFAULT_CONNECT_TIMEOUT - ) - start_time = time.monotonic() try: - async with self._get_httpx_response(request, timeout) as httpx_response: - request.meta["download_latency"] = time.monotonic() - start_time - return await self._read_response(httpx_response, request) + async with client.stream( + request.method, + request.url, + content=request.body, + headers=request.headers.to_tuple_list(), + timeout=timeout, + ) as response: + yield response except httpx.TimeoutException as e: raise DownloadTimeoutError( f"Getting {request.url} took longer than {timeout} seconds." @@ -140,168 +169,58 @@ class HttpxDownloadHandler(BaseHttpDownloadHandler): except httpx.UnsupportedProtocol as e: raise UnsupportedURLSchemeError(str(e)) from e except httpx.ConnectError as e: - error_message = str(e) - if ( - "Name or service not known" in error_message - or "getaddrinfo failed" in error_message - or "nodename nor servname" in error_message - or "Temporary failure in name resolution" in error_message - ): - raise CannotResolveHostError(error_message) from e + if any(isinstance(c, gaierror) for c in _iter_exc_causes(e)): + raise CannotResolveHostError(str(e)) from e raise DownloadConnectionRefusedError(str(e)) from e - except httpx.NetworkError as e: + except httpx.ProxyError as e: + raise DownloadConnectionRefusedError(str(e)) from e + except DOWNLOAD_FAILED_EXCEPTIONS as e: raise DownloadFailedError(str(e)) from e - except httpx.RemoteProtocolError as e: - raise DownloadFailedError(str(e)) from e - - def _warn_unsupported_meta(self, meta: dict[str, Any]) -> None: - if meta.get("bindaddress"): - # configurable only per-client: - # https://github.com/encode/httpx/issues/755#issuecomment-2746121794 - logger.error( - f"The 'bindaddress' request meta key is not supported by" - f" {type(self).__name__} and will be ignored." - ) - if meta.get("proxy"): - # configurable only per-client: - # https://github.com/encode/httpx/issues/486 - logger.error( - f"The 'proxy' request meta key is not supported by" - f" {type(self).__name__} and will be ignored." - ) - - def _get_httpx_response( - self, request: Request, timeout: float - ) -> AbstractAsyncContextManager[httpx.Response]: - return self._client.stream( - request.method, - request.url, - content=request.body, - headers=request.headers.to_tuple_list(), - timeout=timeout, - ) - - async def _read_response( - self, httpx_response: httpx.Response, request: Request - ) -> Response: - maxsize: int = request.meta.get("download_maxsize", self._default_maxsize) - warnsize: int = request.meta.get("download_warnsize", self._default_warnsize) - - content_length = httpx_response.headers.get("Content-Length") - expected_size = int(content_length) if content_length is not None else None - if maxsize and expected_size and expected_size > maxsize: - self._cancel_maxsize(expected_size, maxsize, request, expected=True) - - reached_warnsize = False - if warnsize and expected_size and expected_size > warnsize: - reached_warnsize = True - logger.warning( - get_warnsize_msg(expected_size, warnsize, request, expected=True) - ) - - headers = Headers(httpx_response.headers.multi_items()) - network_stream: AsyncNetworkStream = httpx_response.extensions["network_stream"] - - make_response_base_args: _BaseResponseArgs = { - "status": httpx_response.status_code, - "url": request.url, - "headers": headers, - "ip_address": self._get_server_ip(network_stream), - "protocol": httpx_response.http_version, - } - - self._log_tls_info(network_stream) - - if stop_download := check_stop_download( - signals.headers_received, - self.crawler, - request, - headers=headers, - body_length=expected_size, - ): - return make_response( - **make_response_base_args, - stop_download=stop_download, - ) - - response_body = BytesIO() - bytes_received = 0 - try: - async for chunk in httpx_response.aiter_raw(): - response_body.write(chunk) - bytes_received += len(chunk) - - if stop_download := check_stop_download( - signals.bytes_received, self.crawler, request, data=chunk - ): - return make_response( - **make_response_base_args, - body=response_body.getvalue(), - stop_download=stop_download, - ) - - if maxsize and bytes_received > maxsize: - response_body.truncate(0) - self._cancel_maxsize( - bytes_received, maxsize, request, expected=False - ) - - if warnsize and bytes_received > warnsize and not reached_warnsize: - reached_warnsize = True - logger.warning( - get_warnsize_msg( - bytes_received, warnsize, request, expected=False - ) - ) - except httpx.RemoteProtocolError as e: - # special handling of the dataloss case - if ( - "peer closed connection without sending complete message body" - not in str(e) - ): - raise - fail_on_dataloss: bool = request.meta.get( - "download_fail_on_dataloss", self._fail_on_dataloss - ) - if not fail_on_dataloss: - return make_response( - **make_response_base_args, - body=response_body.getvalue(), - flags=["dataloss"], - ) - self._log_dataloss_warning(request.url) - raise ResponseDataLossError(str(e)) from e - - return make_response( - **make_response_base_args, - body=response_body.getvalue(), - ) @staticmethod - def _get_server_ip(network_stream: AsyncNetworkStream) -> IPv4Address | IPv6Address: - extra_server_addr = network_stream.get_extra_info("server_addr") - return ipaddress.ip_address(extra_server_addr[0]) + def _extract_headers(response: httpx.Response) -> Headers: + return Headers(response.headers.multi_items()) - def _log_tls_info(self, network_stream: AsyncNetworkStream) -> None: - if not self._tls_verbose_logging: - return + @staticmethod + def _build_base_response_args( + response: httpx.Response, + request: Request, + headers: Headers, + ) -> _BaseResponseArgs: + network_stream: AsyncNetworkStream = response.extensions["network_stream"] + server_addr = network_stream.get_extra_info("server_addr") + ip_address = ipaddress.ip_address(server_addr[0]) + ssl_object = network_stream.get_extra_info("ssl_object") + if isinstance(ssl_object, ssl.SSLObject): + cert = ssl_object.getpeercert(binary_form=True) + else: + cert = None + return { + "status": response.status_code, + "url": request.url, + "headers": headers, + "certificate": cert, + "ip_address": ip_address, + "protocol": response.http_version, + } + + @staticmethod + def _iter_body_chunks(response: httpx.Response) -> AsyncIterator[bytes]: + return response.aiter_raw() + + @staticmethod + def _is_dataloss_exception(exc: Exception) -> bool: + return isinstance( + exc, httpx.RemoteProtocolError + ) and "peer closed connection without sending complete message body" in str(exc) + + def _log_tls_info(self, response: httpx.Response, request: Request) -> None: + network_stream: AsyncNetworkStream = response.extensions["network_stream"] extra_ssl_object = network_stream.get_extra_info("ssl_object") if isinstance(extra_ssl_object, ssl.SSLObject): _log_sslobj_debug_info(extra_ssl_object) - def _log_dataloss_warning(self, url: str) -> None: - if self._fail_on_dataloss_warned: - return - logger.warning(get_dataloss_msg(url)) - self._fail_on_dataloss_warned = True - - @staticmethod - def _cancel_maxsize( - size: int, limit: int, request: Request, *, expected: bool - ) -> NoReturn: - warning_msg = get_maxsize_msg(size, limit, request, expected=expected) - logger.warning(warning_msg) - raise DownloadCancelledError(warning_msg) - async def close(self) -> None: - await self._client.aclose() + await self._default_client.aclose() + for client in self._proxy_clients.values(): + await client.aclose() diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index e6451cdd7..85a5ae0d7 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -41,7 +41,6 @@ from scrapy.exceptions import ( ) from scrapy.http import Headers, Response from scrapy.utils._download_handlers import ( - BaseHttpDownloadHandler, check_stop_download, get_dataloss_msg, get_maxsize_msg, @@ -57,6 +56,8 @@ from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.ssl import _log_ssl_conn_debug_info from scrapy.utils.url import add_http_if_no_scheme +from ._base_http import BaseHttpDownloadHandler + if TYPE_CHECKING: from twisted.internet.base import ReactorBase from twisted.internet.interfaces import IConsumer @@ -111,7 +112,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler): "download_warnsize", "DOWNLOAD_WARNSIZE" ) - agent = ScrapyAgent( + agent = _ScrapyAgent( contextFactory=self._contextFactory, bindAddress=self._bind_address, pool=self._pool, @@ -161,7 +162,7 @@ class TunnelError(Exception): """An HTTP CONNECT tunnel could not be established by the proxy.""" -class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): +class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): """An endpoint that tunnels through proxies to allow HTTPS downloads. To accomplish that, this endpoint sends an HTTP CONNECT to the proxy. The HTTP CONNECT is always sent when using this endpoint, I think this could @@ -197,7 +198,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def requestTunnel(self, protocol: Protocol) -> Protocol: """Asks the proxy to open a tunnel.""" assert protocol.transport - tunnelReq = tunnel_request_data( + tunnelReq = _tunnel_request_data( self._tunneledHost, self._tunneledPort, self._proxyAuthHeader ) protocol.transport.write(tunnelReq) @@ -221,7 +222,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): if b"\r\n\r\n" not in self._connectBuffer: return self._protocol.dataReceived = self._protocolDataReceived # type: ignore[method-assign] - respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) + respm = _TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) if respm and int(respm.group("status")) == 200: # set proper Server Name Indication extension sslOptions = self._contextFactory.creatorForNetloc( # type: ignore[call-arg,misc] @@ -258,18 +259,18 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): return self._tunnelReadyDeferred -def tunnel_request_data( +def _tunnel_request_data( host: str, port: int, proxy_auth_header: bytes | None = None ) -> bytes: r""" Return binary content of a CONNECT request. >>> from scrapy.utils.python import to_unicode as s - >>> s(tunnel_request_data("example.com", 8080)) + >>> s(_tunnel_request_data("example.com", 8080)) 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n' - >>> s(tunnel_request_data("example.com", 8080, b"123")) + >>> s(_tunnel_request_data("example.com", 8080, b"123")) 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n' - >>> s(tunnel_request_data(b"example.com", "8090")) + >>> s(_tunnel_request_data(b"example.com", "8090")) 'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n' """ host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port)) @@ -281,7 +282,7 @@ def tunnel_request_data( return tunnel_req -class TunnelingAgent(Agent): +class _TunnelingAgent(Agent): """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS downloads. It may look strange that we have chosen to subclass Agent and not ProxyAgent but consider that after the tunnel is opened the proxy is @@ -303,8 +304,8 @@ class TunnelingAgent(Agent): self._proxyConf: tuple[str, int, bytes | None] = proxyConf self._contextFactory: IPolicyForHTTPS = contextFactory - def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint: - return TunnelingTCP4ClientEndpoint( + def _getEndpoint(self, uri: URI) -> _TunnelingTCP4ClientEndpoint: + return _TunnelingTCP4ClientEndpoint( reactor=self._reactor, host=uri.host, port=uri.port, @@ -381,10 +382,7 @@ class _ScrapyProxyAgent(Agent): ) -class ScrapyAgent: - _Agent = Agent - _TunnelingAgent = TunnelingAgent - +class _ScrapyAgent: def __init__( self, *, @@ -430,7 +428,7 @@ class ScrapyAgent: assert proxy_host is not None proxyAuth = request.headers.get(b"Proxy-Authorization", None) proxyConf = (proxy_host, proxy_port, proxyAuth) - return self._TunnelingAgent( + return _TunnelingAgent( reactor=reactor, proxyConf=proxyConf, contextFactory=self._contextFactory, @@ -447,7 +445,7 @@ class ScrapyAgent: pool=self._pool, ) - return self._Agent( # type: ignore[no-untyped-call] + return Agent( reactor=reactor, contextFactory=self._contextFactory, connectTimeout=timeout, @@ -465,7 +463,7 @@ class ScrapyAgent: url = urldefrag(request.url)[0] method = to_bytes(request.method) headers = TxHeaders(request.headers) - if isinstance(agent, self._TunnelingAgent): + if isinstance(agent, _TunnelingAgent): headers.removeHeader(b"Proxy-Authorization") bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = monotonic() diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py index b28bcfd89..f60c58d1b 100644 --- a/scrapy/core/downloader/handlers/http2.py +++ b/scrapy/core/downloader/handlers/http2.py @@ -49,7 +49,7 @@ class H2DownloadHandler(BaseDownloadHandler): raise UnsupportedURLSchemeError( f"{type(self).__name__} doesn't support plain HTTP." ) - agent = ScrapyH2Agent( + agent = _ScrapyH2Agent( context_factory=self._context_factory, pool=self._pool, bind_address=self._bind_address, @@ -65,9 +65,7 @@ class H2DownloadHandler(BaseDownloadHandler): self._pool.close_connections() -class ScrapyH2Agent: - _Agent = H2Agent - +class _ScrapyH2Agent: def __init__( self, context_factory: IPolicyForHTTPS, @@ -89,7 +87,7 @@ class ScrapyH2Agent: raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.") bind_address = request.meta.get("bindaddress") or self._bind_address bind_address = normalize_bind_address(bind_address) - return self._Agent( + return H2Agent( reactor=reactor, context_factory=self._context_factory, connect_timeout=timeout, diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 19a1e8503..c6660ff48 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any, cast from scrapy.core.downloader.handlers.base import BaseDownloadHandler from scrapy.exceptions import NotConfigured @@ -9,6 +9,8 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import build_from_crawler, load_object if TYPE_CHECKING: + from collections.abc import Mapping + from scrapy import Request from scrapy.crawler import Crawler from scrapy.http import Response @@ -59,7 +61,7 @@ class S3DownloadHandler(BaseDownloadHandler): awsrequest = botocore.awsrequest.AWSRequest( method=request.method, url=f"{scheme}://s3.amazonaws.com/{bucket}{path}", - headers=request.headers.to_unicode_dict(), + headers=cast("Mapping[str, Any]", request.headers.to_unicode_dict()), data=request.body, ) assert self._signer diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py index 6685d90af..ab74e22a4 100644 --- a/scrapy/core/downloader/middleware.py +++ b/scrapy/core/downloader/middleware.py @@ -8,7 +8,7 @@ from __future__ import annotations import warnings from functools import wraps -from typing import TYPE_CHECKING, Any, cast +from typing import TYPE_CHECKING, Any from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Request, Response @@ -87,7 +87,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): result = await self._process_exception(ex, request) return await self._process_response(result, request) - def _handle_mw_method(self, method: Callable, **kwargs: Any) -> Any: + def _handle_mw_method(self, method: Callable[..., Any], **kwargs: Any) -> Any: if method in self._mw_methods_requiring_spider: kwargs["spider"] = self._spider @@ -99,7 +99,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): download_func: Callable[[Request], Coroutine[Any, Any, Response]], ) -> Response | Request: for method in self.methods["process_request"]: - method = cast("Callable", method) + assert method is not None response = await ensure_awaitable( self._handle_mw_method(method, request=request), _warn=global_object_name(method), @@ -122,7 +122,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): return response for method in self.methods["process_response"]: - method = cast("Callable", method) + assert method is not None response = await ensure_awaitable( self._handle_mw_method(method, request=request, response=response), _warn=global_object_name(method), @@ -141,7 +141,7 @@ class DownloaderMiddlewareManager(MiddlewareManager): self, exception: Exception, request: Request | Response ) -> Response | Request: for method in self.methods["process_exception"]: - method = cast("Callable", method) + assert method is not None response = await ensure_awaitable( self._handle_mw_method(method, request=request, exception=exception), _warn=global_object_name(method), diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 779c05daa..3b903b39d 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +import warnings from typing import TYPE_CHECKING, Any from OpenSSL import SSL @@ -18,8 +19,9 @@ from service_identity.pyopenssl import ( verify_ip_address, ) from twisted.internet._sslverify import ClientTLSOptions -from twisted.internet.ssl import AcceptableCiphers +from twisted.internet.ssl import AcceptableCiphers, TLSVersion +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.deprecate import create_deprecated_class if TYPE_CHECKING: @@ -32,17 +34,37 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -METHOD_TLS = "TLS" -METHOD_TLSv10 = "TLSv1.0" -METHOD_TLSv11 = "TLSv1.1" -METHOD_TLSv12 = "TLSv1.2" +_openssl_methods: dict[str, int] = { + "TLS": SSL.SSLv23_METHOD, # protocol negotiation (recommended) + "TLSv1.0": SSL.TLSv1_METHOD, # TLS 1.0 only + "TLSv1.1": SSL.TLSv1_1_METHOD, # TLS 1.1 only + "TLSv1.2": SSL.TLSv1_2_METHOD, # TLS 1.2 only +} -openssl_methods: dict[str, int] = { - METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) - METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only - METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only - METHOD_TLSv12: SSL.TLSv1_2_METHOD, # TLS 1.2 only +def __getattr__(name: str) -> Any: + deprecated = { + "METHOD_TLS": "TLS", + "METHOD_TLSv10": "TLSv1.0", + "METHOD_TLSv11": "TLSv1.1", + "METHOD_TLSv12": "TLSv1.2", + "openssl_methods": _openssl_methods, + } + if name in deprecated: + warnings.warn( + f"scrapy.core.downloader.tls.{name} is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return deprecated[name] + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +_TWISTED_VERSION_MAP: dict[str, TLSVersion] = { + "TLSv1.0": TLSVersion.TLSv1_0, + "TLSv1.1": TLSVersion.TLSv1_1, + "TLSv1.2": TLSVersion.TLSv1_2, + "TLSv1.3": TLSVersion.TLSv1_3, } @@ -107,34 +129,23 @@ class _ScrapyClientTLSOptions26(ClientTLSOptions): logging warnings. Instances of this class are returned from - :class:`.ScrapyClientContextFactory`. + :class:`._ScrapyClientContextFactory`. This class is used on Twisted 26.4.0 and newer. """ - def __init__( - self, - createConnection: Callable[[TLSMemoryBIOProtocol], SSL.Connection], - hostname: str, - verbose_logging: bool = False, - ): - super().__init__(createConnection, hostname) - self.verbose_logging: bool = verbose_logging - def clientConnectionForTLS( self, tlsProtocol: TLSMemoryBIOProtocol ) -> SSL.Connection: """This method is needed to override the verify callback.""" conn = super().clientConnectionForTLS(tlsProtocol) - callback = self._verifyCB( - self._hostnameIsDnsName, self._hostnameASCII, self.verbose_logging - ) + callback = self._verifyCB(self._hostnameIsDnsName, self._hostnameASCII) conn.set_verify(SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, callback) return conn @staticmethod def _verifyCB( - hostIsDNS: bool, hostnameASCII: str, verbose_logging: bool + hostIsDNS: bool, hostnameASCII: str ) -> Callable[[SSL.Connection, X509, int, int, int], bool]: svcid: ServiceID = ( DNS_ID(hostnameASCII) if hostIsDNS else IPAddress_ID(hostnameASCII) @@ -145,7 +156,7 @@ class _ScrapyClientTLSOptions26(ClientTLSOptions): ) -> bool: if depth != 0: # We are only verifying the leaf certificate. - return bool(ok) + return True try: verify_service_identity(extract_patterns(cert), [svcid], []) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index e756d27eb..466ce656d 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -125,7 +125,7 @@ class Scraper: def _check_deprecated_itemproc_method(self, method: str) -> None: itemproc_cls = type(self.itemproc) - if not hasattr(self.itemproc, "process_item_async"): + if not hasattr(self.itemproc, f"{method}_async"): warnings.warn( f"{global_object_name(itemproc_cls)} doesn't define a {method}_async() method," f" this is deprecated and the method will be required in future Scrapy versions.", diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index d48a65967..fbc6f2530 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -9,29 +9,28 @@ from __future__ import annotations import logging from collections.abc import AsyncIterator, Callable, Coroutine, Iterable from functools import wraps -from inspect import isasyncgenfunction, iscoroutine +from inspect import isasyncgenfunction from itertools import islice -from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast +from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar from warnings import warn -from twisted.internet.defer import Deferred, inlineCallbacks from twisted.python.failure import Failure from scrapy import Request, Spider from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Response from scrapy.middleware import MiddlewareManager -from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.asyncgen import as_async_generator from scrapy.utils.conf import build_component_list from scrapy.utils.defer import ( _defer_sleep_async, deferred_from_coro, maybe_deferred_to_future, ) -from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_name +from scrapy.utils.python import MutableAsyncChain, global_object_name if TYPE_CHECKING: - from collections.abc import Generator + from twisted.internet.defer import Deferred from scrapy.settings import BaseSettings @@ -46,10 +45,6 @@ ScrapeFunc: TypeAlias = Callable[ ] -def _isiterable(o: Any) -> bool: - return isinstance(o, (Iterable, AsyncIterator)) - - class SpiderMiddlewareManager(MiddlewareManager): component_name = "spider middleware" @@ -67,13 +62,10 @@ class SpiderMiddlewareManager(MiddlewareManager): if hasattr(mw, "process_start"): self.methods["process_start"].appendleft(mw.process_start) - process_spider_output = self._get_async_method_pair(mw, "process_spider_output") + process_spider_output = self._get_process_spider_output(mw) self.methods["process_spider_output"].appendleft(process_spider_output) - if callable(process_spider_output): + if process_spider_output is not None: self._check_mw_method_spider_arg(process_spider_output) - elif isinstance(process_spider_output, tuple): - for m in process_spider_output: - self._check_mw_method_spider_arg(m) process_spider_exception = getattr(mw, "process_spider_exception", None) self.methods["process_spider_exception"].appendleft(process_spider_exception) @@ -87,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager): request: Request, ) -> Iterable[_T] | AsyncIterator[_T]: for method in self.methods["process_spider_input"]: - method = cast("Callable", method) + assert method is not None try: if method in self._mw_methods_requiring_spider: result = method(response=response, spider=self._spider) @@ -105,48 +97,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return await scrape_func(Failure(), request) return await scrape_func(response, request) - def _evaluate_iterable( - self, - response: Response, - iterable: Iterable[_T] | AsyncIterator[_T], - exception_processor_index: int, - recover_to: MutableChain[_T] | MutableAsyncChain[_T], - ) -> Iterable[_T] | AsyncIterator[_T]: - - if isinstance(iterable, AsyncIterator): - return self._process_async( - response, - iterable, - exception_processor_index, - cast("MutableAsyncChain[_T]", recover_to), - ) - return self._process_sync( - response, - iterable, - exception_processor_index, - cast("MutableChain[_T]", recover_to), - ) - - def _process_sync( - self, - response: Response, - iterable: Iterable[_T], - exception_processor_index: int, - recover_to: MutableChain[_T], - ) -> Iterable[_T]: - try: - yield from iterable - except Exception as ex: - exception_result = cast( - "Failure | MutableChain[_T]", - self._process_spider_exception(response, ex, exception_processor_index), - ) - if isinstance(exception_result, Failure): - raise - assert isinstance(recover_to, MutableChain) - recover_to.extend(exception_result) - - async def _process_async( + async def _evaluate_iterable( self, response: Response, iterable: AsyncIterator[_T], @@ -157,13 +108,9 @@ class SpiderMiddlewareManager(MiddlewareManager): async for r in iterable: yield r except Exception as ex: - exception_result = cast( - "Failure | MutableAsyncChain[_T]", - self._process_spider_exception(response, ex, exception_processor_index), + exception_result: MutableAsyncChain[_T] = self._process_spider_exception( + response, ex, exception_processor_index ) - if isinstance(exception_result, Failure): - raise - assert isinstance(recover_to, MutableAsyncChain) recover_to.extend(exception_result) def _process_spider_exception( @@ -171,7 +118,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, exception: Exception, start_index: int = 0, - ) -> MutableChain[_T] | MutableAsyncChain[_T]: + ) -> MutableAsyncChain[_T]: # don't handle _InvalidOutput exception if isinstance(exception, _InvalidOutput): raise exception @@ -181,28 +128,18 @@ class SpiderMiddlewareManager(MiddlewareManager): for method_index, method in enumerate(method_list, start=start_index): if method is None: continue - method = cast("Callable", method) if method in self._mw_methods_requiring_spider: result = method( response=response, exception=exception, spider=self._spider ) else: result = method(response=response, exception=exception) - if _isiterable(result): + if isinstance(result, (Iterable, AsyncIterator)): # stop exception handling by handing control over to the # process_spider_output chain if an iterable has been returned - dfd: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = ( - self._process_spider_output(response, result, method_index + 1) - ) - # _process_spider_output() returns a Deferred only because of downgrading so this can be - # simplified when downgrading is removed. - if dfd.called: - # the result is available immediately if _process_spider_output didn't do downgrading - return cast("MutableChain[_T] | MutableAsyncChain[_T]", dfd.result) - # we forbid waiting here because otherwise we would need to return a deferred from - # _process_spider_exception too, which complicates the architecture - msg = f"Async iterable returned from {global_object_name(method)} cannot be downgraded" - raise _InvalidOutput(msg) + if isinstance(result, Iterable): + result = as_async_generator(result) + return self._process_spider_output(response, result, method_index + 1) if result is None: continue msg = ( @@ -212,124 +149,35 @@ class SpiderMiddlewareManager(MiddlewareManager): raise _InvalidOutput(msg) raise exception - # This method cannot be made async def, as _process_spider_exception relies on the Deferred result - # being available immediately which doesn't work when it's a wrapped coroutine. - # It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed. - @inlineCallbacks - def _process_spider_output( # noqa: PLR0912 + def _process_spider_output( self, response: Response, - result: Iterable[_T] | AsyncIterator[_T], + result: AsyncIterator[_T], start_index: int = 0, - ) -> Generator[Deferred[Any], Any, MutableChain[_T] | MutableAsyncChain[_T]]: + ) -> MutableAsyncChain[_T]: # items in this iterable do not need to go through the process_spider_output # chain, they went through it already from the process_spider_exception method - recovered: MutableChain[_T] | MutableAsyncChain[_T] - last_result_is_async = isinstance(result, AsyncIterator) - recovered = MutableAsyncChain() if last_result_is_async else MutableChain() - - # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async. - # 1. def foo. Sync iterables are passed as is, async ones are downgraded. - # 2. async def foo. Sync iterables are upgraded, async ones are passed as is. - # 3. def foo + async def foo_async. Iterables are passed to the respective method. - # Storing methods and method tuples in the same list is weird but we should be able to roll this back - # when we drop this compatibility feature. - + recovered: MutableAsyncChain[_T] = MutableAsyncChain() method_list = islice(self.methods["process_spider_output"], start_index, None) - for method_index, method_pair in enumerate(method_list, start=start_index): - if method_pair is None: + for method_index, method in enumerate(method_list, start=start_index): + if method is None: continue - need_upgrade = need_downgrade = False - if isinstance(method_pair, tuple): - # This tuple handling is only needed until _async compatibility methods are removed. - method_sync, method_async = method_pair - method = method_async if last_result_is_async else method_sync + if method in self._mw_methods_requiring_spider: + result = method(response=response, result=result, spider=self._spider) else: - method = method_pair - if not last_result_is_async and isasyncgenfunction(method): - need_upgrade = True - elif last_result_is_async and not isasyncgenfunction(method): - need_downgrade = True - try: - if need_upgrade: - # Iterable -> AsyncIterator - result = as_async_generator(result) - elif need_downgrade: - logger.warning( - f"Async iterable passed to {global_object_name(method)} was" - f" downgraded to a non-async one. This is deprecated and will" - f" stop working in a future version of Scrapy. Please see" - f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users" - f" for more information." - ) - assert isinstance(result, AsyncIterator) - # AsyncIterator -> Iterable - result = yield deferred_from_coro(collect_asyncgen(result)) - if isinstance(recovered, AsyncIterator): - recovered_collected = yield deferred_from_coro( - collect_asyncgen(recovered) - ) - recovered = MutableChain(recovered_collected) - # might fail directly if the output value is not a generator - if method in self._mw_methods_requiring_spider: - result = method( - response=response, result=result, spider=self._spider - ) - else: - result = method(response=response, result=result) - except Exception as ex: - exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = ( - self._process_spider_exception(response, ex, method_index + 1) - ) - if isinstance(exception_result, Failure): - raise - return exception_result - if _isiterable(result): - result = self._evaluate_iterable( - response, result, method_index + 1, recovered - ) - else: - if iscoroutine(result): - result.close() # Silence warning about not awaiting - msg = ( - f"{global_object_name(method)} must be an asynchronous " - f"generator (i.e. use yield)" - ) - else: - msg = ( - f"{global_object_name(method)} must return an iterable, got " - f"{type(result)}" - ) - raise _InvalidOutput(msg) - last_result_is_async = isinstance(result, AsyncIterator) - - if last_result_is_async: - return MutableAsyncChain(result, recovered) - return MutableChain(result, recovered) # type: ignore[arg-type] + result = method(response=response, result=result) + result = self._evaluate_iterable( + response, result, method_index + 1, recovered + ) + return MutableAsyncChain(result, recovered) async def _process_callback_output( - self, - response: Response, - result: Iterable[_T] | AsyncIterator[_T], - ) -> MutableChain[_T] | MutableAsyncChain[_T]: - recovered: MutableChain[_T] | MutableAsyncChain[_T] - if isinstance(result, AsyncIterator): - recovered = MutableAsyncChain() - else: - recovered = MutableChain() + self, response: Response, result: AsyncIterator[_T] + ) -> MutableAsyncChain[_T]: + recovered: MutableAsyncChain[_T] = MutableAsyncChain() result = self._evaluate_iterable(response, result, 0, recovered) - result = await maybe_deferred_to_future( - cast( - "Deferred[Iterable[_T] | AsyncIterator[_T]]", - self._process_spider_output(response, result), - ) - ) - if isinstance(result, AsyncIterator): - return MutableAsyncChain(result, recovered) - if isinstance(recovered, AsyncIterator): - recovered_collected = await collect_asyncgen(recovered) - recovered = MutableChain(recovered_collected) - return MutableChain(result, recovered) + result = self._process_spider_output(response, result) + return MutableAsyncChain(result, recovered) def scrape_response( self, @@ -340,7 +188,7 @@ class SpiderMiddlewareManager(MiddlewareManager): response: Response, request: Request, spider: Spider, - ) -> Deferred[MutableChain[_T] | MutableAsyncChain[_T]]: # pragma: no cover + ) -> Deferred[MutableAsyncChain[_T]]: # pragma: no cover warn( "SpiderMiddlewareManager.scrape_response() is deprecated, use scrape_response_async() instead", ScrapyDeprecationWarning, @@ -363,7 +211,7 @@ class SpiderMiddlewareManager(MiddlewareManager): scrape_func: ScrapeFunc[_T], response: Response, request: Request, - ) -> MutableChain[_T] | MutableAsyncChain[_T]: + ) -> MutableAsyncChain[_T]: if not self.crawler: raise RuntimeError( "scrape_response_async() called on a SpiderMiddlewareManager" @@ -373,7 +221,8 @@ class SpiderMiddlewareManager(MiddlewareManager): it: Iterable[_T] | AsyncIterator[_T] = await self._process_spider_input( scrape_func, response, request ) - return await self._process_callback_output(response, it) + ait = it if isinstance(it, AsyncIterator) else as_async_generator(it) + return await self._process_callback_output(response, ait) except Exception as ex: await _defer_sleep_async() return self._process_spider_exception(response, ex) @@ -399,27 +248,27 @@ class SpiderMiddlewareManager(MiddlewareManager): # This method is only needed until _async compatibility methods are removed. @staticmethod - def _get_async_method_pair( - mw: Any, methodname: str - ) -> Callable | tuple[Callable, Callable] | None: - normal_method: Callable | None = getattr(mw, methodname, None) - methodname_async = methodname + "_async" - async_method: Callable | None = getattr(mw, methodname_async, None) + def _get_process_spider_output(mw: Any) -> Callable[..., Any] | None: + normal_method: Callable[..., Any] | None = getattr( + mw, "process_spider_output", None + ) + async_method: Callable[..., Any] | None = getattr( + mw, "process_spider_output_async", None + ) if not async_method: if normal_method and not isasyncgenfunction(normal_method): - logger.warning( + raise TypeError( f"Middleware {global_object_name(mw.__class__)} doesn't support" - f" asynchronous spider output, this is deprecated and will stop" - f" working in a future version of Scrapy. The middleware should" - f" be updated to support it. Please see" - f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users" - f" for more information." + f" asynchronous spider output. Its process_spider_output() method" + f" should be an async generator function or it should additionally" + f" define a process_spider_output_async() method." ) return normal_method if not normal_method: logger.error( - f"Middleware {global_object_name(mw.__class__)} has {methodname_async} " - f"without {methodname}, skipping this method." + f"Middleware {global_object_name(mw.__class__)} has" + f" process_spider_output_async() without process_spider_output()," + f" skipping this method. Please rename it to process_spider_output()." ) return None if not isasyncgenfunction(async_method): @@ -431,8 +280,8 @@ class SpiderMiddlewareManager(MiddlewareManager): if isasyncgenfunction(normal_method): logger.error( f"{global_object_name(normal_method)} is an async " - f"generator function while {methodname_async} exists, " - f"skipping both methods." + f"generator function while process_spider_output_async() exists, " + f"skipping both methods. Please remove process_spider_output_async()." ) return None - return normal_method, async_method + return async_method diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index d38b4b9db..5f125cae4 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -12,7 +12,7 @@ once the spider has finished crawling all regular (non-failed) pages. from __future__ import annotations -from logging import Logger, getLogger +from logging import Logger, getLevelName, getLogger from typing import TYPE_CHECKING from scrapy.exceptions import NotConfigured @@ -43,6 +43,7 @@ def get_retry_request( max_retry_times: int | None = None, priority_adjust: int | None = None, logger: Logger = retry_logger, + give_up_log_level: int | str | None = None, stats_base_key: str = "retry", ) -> Request | None: """ @@ -51,14 +52,16 @@ def get_retry_request( exhausted. For example, in a :class:`~scrapy.Spider` callback, you could use it as - follows:: + follows: + + .. code-block:: python def parse(self, response): if not response.text: new_request_or_none = get_retry_request( response.request, spider=self, - reason='empty', + reason="empty", ) return new_request_or_none @@ -82,6 +85,10 @@ def get_retry_request( *logger* is the logging.Logger object to be used when logging messages + *give_up_log_level* is the :ref:`logging level ` used for the + message logged when a request exceeds its retries. See + :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details. + *stats_base_key* is a string to be used as the base key for the retry-related job stats """ @@ -114,8 +121,16 @@ def get_retry_request( stats.inc_value(f"{stats_base_key}/count") stats.inc_value(f"{stats_base_key}/reason_count/{reason}") return new_request + if give_up_log_level is None: + give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"] + if isinstance(give_up_log_level, str): + level = getLevelName(give_up_log_level) + if not isinstance(level, int): + raise ValueError(f"Invalid give-up log level: {give_up_log_level!r}") + give_up_log_level = level stats.inc_value(f"{stats_base_key}/max_reached") - logger.error( + logger.log( + give_up_log_level, "Gave up retrying %(request)s (failed %(retry_times)d times): %(reason)s", {"request": request, "retry_times": retry_times, "reason": reason}, extra={"spider": spider}, @@ -132,6 +147,7 @@ class RetryMiddleware: self.max_retry_times = settings.getint("RETRY_TIMES") self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")} self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST") + self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"] self.exceptions_to_retry = tuple( load_object(x) if isinstance(x, str) else x for x in settings.getlist("RETRY_EXCEPTIONS") @@ -175,6 +191,9 @@ class RetryMiddleware: ) -> Request | None: max_retry_times = request.meta.get("max_retry_times", self.max_retry_times) priority_adjust = request.meta.get("priority_adjust", self.priority_adjust) + give_up_log_level = request.meta.get( + "give_up_log_level", self.give_up_log_level + ) assert self.crawler.spider return get_retry_request( request, @@ -182,4 +201,5 @@ class RetryMiddleware: spider=self.crawler.spider, max_retry_times=max_retry_times, priority_adjust=priority_adjust, + give_up_log_level=give_up_log_level, ) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index d0430d0db..8029f85c9 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -306,12 +306,15 @@ class GCSFeedStorage(BlockingFeedStorage): def _store_in_thread(self, file: IO[bytes]) -> None: file.seek(0) - from google.cloud.storage import Client # noqa: PLC0415 + try: + from google.cloud.storage import Client # noqa: PLC0415 - client = Client(project=self.project_id) - bucket = client.get_bucket(self.bucket_name) - blob = bucket.blob(self.blob_name) - blob.upload_from_file(file, predefined_acl=self.acl) + client = Client(project=self.project_id) + bucket = client.get_bucket(self.bucket_name) + blob = bucket.blob(self.blob_name) + blob.upload_from_file(file, predefined_acl=self.acl) + finally: + file.close() class FTPFeedStorage(BlockingFeedStorage): diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 86b066a38..dbb79b02d 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -313,7 +313,9 @@ class FilesystemCacheStorage: self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS") self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP") # https://github.com/python/mypy/issues/10740 - self._open: Callable[Concatenate[str | os.PathLike, str, ...], IO[bytes]] = ( + self._open: Callable[ + Concatenate[str | os.PathLike[str], str, ...], IO[bytes] + ] = ( gzip.open if self.use_gzip else open # type: ignore[assignment] ) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 00042e093..7db648a45 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -138,6 +138,7 @@ class Request(object_ref): ) -> None: self._encoding: str = encoding # this one has to be set first self.method: str = str(method).upper() + self._meta: dict[str, Any] | None = dict(meta) if meta else None self._set_url(url) self._set_body(body) if not isinstance(priority, int): @@ -232,7 +233,6 @@ class Request(object_ref): #: default. See :meth:`~scrapy.Spider.start`. self.dont_filter: bool = dont_filter - self._meta: dict[str, Any] | None = dict(meta) if meta else None self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else None self._flags: list[str] | None = list(flags) if flags else None @@ -252,11 +252,17 @@ class Request(object_ref): def url(self) -> str: return self._url + def _url_is_verbatim(self) -> bool: + return bool(self._meta and self._meta.get("verbatim_url")) + def _set_url(self, url: str) -> None: if not isinstance(url, str): raise TypeError(f"Request url must be str, got {type(url).__name__}") - self._url = safe_url_string(url, self.encoding) + if self._url_is_verbatim(): + self._url = url + else: + self._url = safe_url_string(url, self.encoding) if ( "://" not in self._url diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index a80ea3da8..09b1c8b32 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -20,7 +20,6 @@ if TYPE_CHECKING: from collections.abc import Callable, Iterable, Mapping from ipaddress import IPv4Address, IPv6Address - from twisted.internet.ssl import Certificate from twisted.python.failure import Failure # typing.Self requires Python 3.11 @@ -77,7 +76,7 @@ class Response(object_ref): body: bytes = b"", flags: list[str] | None = None, request: Request | None = None, - certificate: Certificate | None = None, + certificate: Any = None, ip_address: IPv4Address | IPv6Address | None = None, protocol: str | None = None, ): @@ -87,7 +86,7 @@ class Response(object_ref): self._set_url(url) self.request: Request | None = request self._flags: list[str] | None = list(flags) if flags else None - self.certificate: Certificate | None = certificate + self.certificate: Any = certificate self.ip_address: IPv4Address | IPv6Address | None = ip_address self.protocol: str | None = protocol diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 13853f64d..6876e35e8 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -220,7 +220,7 @@ class TextResponse(Response): def follow_all( self, - urls: Iterable[str | Link] | parsel.SelectorList | None = None, + urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None, callback: CallbackT | None = None, method: str = "GET", headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None, diff --git a/scrapy/middleware.py b/scrapy/middleware.py index c4e0ce16e..37ec0f583 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -50,11 +50,8 @@ class MiddlewareManager(ABC): ) self.middlewares: tuple[Any, ...] = middlewares # Only process_spider_output and process_spider_exception can be None. - # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed. - self.methods: dict[str, deque[Callable | tuple[Callable, Callable] | None]] = ( - defaultdict(deque) - ) - self._mw_methods_requiring_spider: set[Callable] = set() + self.methods: dict[str, deque[Callable[..., Any] | None]] = defaultdict(deque) + self._mw_methods_requiring_spider: set[Callable[..., Any]] = set() for mw in middlewares: self._add_middleware(mw) @@ -119,7 +116,7 @@ class MiddlewareManager(ABC): def _add_middleware(self, mw: Any) -> None: # noqa: B027 pass - def _check_mw_method_spider_arg(self, method: Callable) -> None: + def _check_mw_method_spider_arg(self, method: Callable[..., Any]) -> None: if argument_is_required(method, "spider"): warnings.warn( f"{method.__qualname__}() requires a spider argument," diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index d4671fd35..be298e9b1 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -16,10 +16,6 @@ from scrapy.utils.python import global_object_name logger = getLogger(__name__) -# The key types are restricted in BaseSettings._get_key() to ones supported by JSON, -# see https://github.com/scrapy/scrapy/issues/5383. -_SettingsKey: TypeAlias = bool | float | int | str | None - if TYPE_CHECKING: from types import ModuleType @@ -29,7 +25,7 @@ if TYPE_CHECKING: # typing.Self requires Python 3.11 from typing_extensions import Self - _SettingsInput: TypeAlias = SupportsItems[_SettingsKey, Any] | str | None + _SettingsInput: TypeAlias = SupportsItems[str, Any] | str | None SETTINGS_PRIORITIES: dict[str, int] = { @@ -80,7 +76,7 @@ class SettingsAttribute: return f"" -class BaseSettings(MutableMapping[_SettingsKey, Any]): +class BaseSettings(MutableMapping[str, Any]): """ Instances of this class behave like dictionaries, but store priorities along with their ``(key, value)`` pairs, and can be frozen (i.e. marked @@ -106,11 +102,11 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): def __init__(self, values: _SettingsInput = None, priority: int | str = "project"): self.frozen: bool = False - self.attributes: dict[_SettingsKey, SettingsAttribute] = {} + self.attributes: dict[str, SettingsAttribute] = {} if values: self.update(values, priority) - def __getitem__(self, opt_name: _SettingsKey) -> Any: + def __getitem__(self, opt_name: str) -> Any: if opt_name not in self: return None return self.attributes[opt_name].value @@ -118,7 +114,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): def __contains__(self, name: Any) -> bool: return name in self.attributes - def add_to_list(self, name: _SettingsKey, item: Any) -> None: + def add_to_list(self, name: str, item: Any) -> None: """Append *item* to the :class:`list` setting with the specified *name* if *item* is not already in that list. @@ -129,7 +125,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): if item not in value: self.set(name, [*value, item], self.getpriority(name) or 0) - def remove_from_list(self, name: _SettingsKey, item: Any) -> None: + def remove_from_list(self, name: str, item: Any) -> None: """Remove *item* from the :class:`list` setting with the specified *name*. @@ -143,7 +139,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).") self.set(name, [v for v in value if v != item], self.getpriority(name) or 0) - def get(self, name: _SettingsKey, default: Any = None) -> Any: # pylint: disable=arguments-renamed + def get(self, name: str, default: Any = None) -> Any: # pylint: disable=arguments-renamed """ Get a setting value without affecting its original type. @@ -172,7 +168,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): return self[name] if self[name] is not None else default - def getbool(self, name: _SettingsKey, default: bool = False) -> bool: + def getbool(self, name: str, default: bool = False) -> bool: """ Get a setting value as a boolean. @@ -202,7 +198,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): "'True'/'False' and 'true'/'false'" ) from None - def getint(self, name: _SettingsKey, default: int = 0) -> int: + def getint(self, name: str, default: int = 0) -> int: """ Get a setting value as an int. @@ -214,7 +210,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): """ return int(self.get(name, default)) - def getfloat(self, name: _SettingsKey, default: float = 0.0) -> float: + def getfloat(self, name: str, default: float = 0.0) -> float: """ Get a setting value as a float. @@ -226,9 +222,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): """ return float(self.get(name, default)) - def getlist( - self, name: _SettingsKey, default: list[Any] | None = None - ) -> list[Any]: + def getlist(self, name: str, default: list[Any] | None = None) -> list[Any]: """ Get a setting value as a list. If the setting original type is a list, a copy of it will be returned. If it's a string it will be split by @@ -251,7 +245,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): return list(value) def getdict( - self, name: _SettingsKey, default: dict[Any, Any] | None = None + self, name: str, default: dict[Any, Any] | None = None ) -> dict[Any, Any]: """ Get a setting value as a dictionary. If the setting original type is a @@ -275,7 +269,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): def getdictorlist( self, - name: _SettingsKey, + name: str, default: dict[Any, Any] | list[Any] | tuple[Any] | None = None, ) -> dict[Any, Any] | list[Any]: """Get a setting value as either a :class:`dict` or a :class:`list`. @@ -322,7 +316,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): ) return copy.deepcopy(value) - def getwithbase(self, name: _SettingsKey) -> BaseSettings: + def getwithbase(self, name: str) -> BaseSettings: """Get a composition of a dictionary-like setting and its ``_BASE`` counterpart. @@ -341,7 +335,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): compbs.update(self[name]) return compbs - def get_component_priority_dict_with_base(self, name: _SettingsKey) -> BaseSettings: + def get_component_priority_dict_with_base(self, name: str) -> BaseSettings: """Get a composition of a component priority dictionary setting and its ``_BASE`` counterpart. @@ -389,7 +383,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): {restore_key(k): v for k, v in result.items() if v is not None} ) - def getpriority(self, name: _SettingsKey) -> int | None: + def getpriority(self, name: str) -> int | None: """ Return the current numerical priority value of a setting, or ``None`` if the given ``name`` does not exist. @@ -414,7 +408,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): def replace_in_component_priority_dict( self, - name: _SettingsKey, + name: str, old_cls: type, new_cls: type, priority: int | None = None, @@ -453,12 +447,10 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): ) self.set(name, component_priority_dict, priority=self.getpriority(name) or 0) - def __setitem__(self, name: _SettingsKey, value: Any) -> None: + def __setitem__(self, name: str, value: Any) -> None: self.set(name, value) - def set( - self, name: _SettingsKey, value: Any, priority: int | str = "project" - ) -> None: + def set(self, name: str, value: Any, priority: int | str = "project") -> None: """ Store a key/value attribute with a given priority. @@ -487,7 +479,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): self.attributes[name].set(value, priority) def set_in_component_priority_dict( - self, name: _SettingsKey, cls: type, priority: int | None + self, name: str, cls: type, priority: int | None ) -> None: """Set the *cls* component in the *name* :ref:`component priority dictionary ` setting with *priority*. @@ -512,7 +504,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): def setdefault( # pylint: disable=arguments-renamed self, - name: _SettingsKey, + name: str, default: Any = None, priority: int | str = "project", ) -> Any: @@ -523,7 +515,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): return self.attributes[name].value def setdefault_in_component_priority_dict( - self, name: _SettingsKey, cls: type, priority: int | None + self, name: str, cls: type, priority: int | None ) -> None: """Set the *cls* component in the *name* :ref:`component priority dictionary ` setting with *priority* @@ -592,7 +584,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): """ self._assert_mutability() if isinstance(values, str): - values = cast("dict[_SettingsKey, Any]", json.loads(values)) + values = cast("dict[str, Any]", json.loads(values)) if values is not None: if isinstance(values, BaseSettings): for name, value in values.items(): @@ -601,7 +593,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): for name, value in values.items(): self.set(name, value, priority) - def delete(self, name: _SettingsKey, priority: int | str = "project") -> None: + def delete(self, name: str, priority: int | str = "project") -> None: if name not in self: raise KeyError(name) self._assert_mutability() @@ -609,7 +601,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): if priority >= cast("int", self.getpriority(name)): del self.attributes[name] - def __delitem__(self, name: _SettingsKey) -> None: + def __delitem__(self, name: str) -> None: self._assert_mutability() del self.attributes[name] @@ -649,26 +641,19 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): copy.freeze() return copy - def __iter__(self) -> Iterator[_SettingsKey]: + def __iter__(self) -> Iterator[str]: return iter(self.attributes) def __len__(self) -> int: return len(self.attributes) - def _to_dict(self) -> dict[_SettingsKey, Any]: + def _to_dict(self) -> dict[str, Any]: return { - self._get_key(k): (v._to_dict() if isinstance(v, BaseSettings) else v) + str(k): (v._to_dict() if isinstance(v, BaseSettings) else v) for k, v in self.items() } - def _get_key(self, key_value: Any) -> _SettingsKey: - return ( - key_value - if isinstance(key_value, (bool, float, int, str, type(None))) - else str(key_value) - ) - - def copy_to_dict(self) -> dict[_SettingsKey, Any]: + def copy_to_dict(self) -> dict[str, Any]: """ Make a copy of current settings and convert to a dict. @@ -691,7 +676,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): else: p.text(pformat(self.copy_to_dict())) - def pop(self, name: _SettingsKey, default: Any = __default) -> Any: # pylint: disable=arguments-renamed + def pop(self, name: str, default: Any = __default) -> Any: # pylint: disable=arguments-renamed try: value = self.attributes[name].value except KeyError: @@ -735,7 +720,7 @@ def iter_default_settings() -> Iterable[tuple[str, Any]]: def overridden_settings( - settings: Mapping[_SettingsKey, Any], + settings: Mapping[str, Any], ) -> Iterable[tuple[str, Any]]: """Return an iterable of the settings that have been overridden""" for name, defvalue in iter_default_settings(): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 5d671feb3..e8a9400d7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -16,22 +16,30 @@ Scrapy developers, if you add a setting here remember to: import sys from importlib import import_module from pathlib import Path +from typing import Any __all__ = [ "ADDONS", - "AJAXCRAWL_ENABLED", - "AJAXCRAWL_MAXSIZE", "ASYNCIO_EVENT_LOOP", "AUTOTHROTTLE_DEBUG", "AUTOTHROTTLE_ENABLED", "AUTOTHROTTLE_MAX_DELAY", "AUTOTHROTTLE_START_DELAY", "AUTOTHROTTLE_TARGET_CONCURRENCY", + "AWS_ACCESS_KEY_ID", + "AWS_ENDPOINT_URL", + "AWS_REGION_NAME", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_USE_SSL", + "AWS_VERIFY", "BOT_NAME", "CLOSESPIDER_ERRORCOUNT", "CLOSESPIDER_ITEMCOUNT", "CLOSESPIDER_PAGECOUNT", + "CLOSESPIDER_PAGECOUNT_NO_ITEM", "CLOSESPIDER_TIMEOUT", + "CLOSESPIDER_TIMEOUT_NO_ITEM", "COMMANDS_MODULE", "COMPRESSION_ENABLED", "CONCURRENT_ITEMS", @@ -63,9 +71,14 @@ __all__ = [ "DOWNLOAD_HANDLERS", "DOWNLOAD_HANDLERS_BASE", "DOWNLOAD_MAXSIZE", + "DOWNLOAD_SLOTS", "DOWNLOAD_TIMEOUT", + "DOWNLOAD_TLS_MAX_VERSION", + "DOWNLOAD_TLS_MIN_VERSION", + "DOWNLOAD_VERIFY_CERTIFICATES", "DOWNLOAD_WARNSIZE", "DUPEFILTER_CLASS", + "DUPEFILTER_DEBUG", "EDITOR", "EXTENSIONS", "EXTENSIONS_BASE", @@ -84,7 +97,9 @@ __all__ = [ "FEED_STORAGE_S3_ACL", "FEED_STORE_EMPTY", "FEED_TEMPDIR", + "FEED_URI", "FEED_URI_PARAMS", + "FILES_STORE", "FILES_STORE_GCS_ACL", "FILES_STORE_S3_ACL", "FORCE_CRAWLER_PROCESS", @@ -104,8 +119,12 @@ __all__ = [ "HTTPCACHE_IGNORE_SCHEMES", "HTTPCACHE_POLICY", "HTTPCACHE_STORAGE", + "HTTPERROR_ALLOWED_CODES", + "HTTPERROR_ALLOW_ALL", "HTTPPROXY_AUTH_ENCODING", "HTTPPROXY_ENABLED", + "HTTPX_HTTP2_ENABLED", + "IMAGES_STORE", "IMAGES_STORE_GCS_ACL", "IMAGES_STORE_S3_ACL", "ITEM_PIPELINES", @@ -128,6 +147,8 @@ __all__ = [ "MAIL_HOST", "MAIL_PASS", "MAIL_PORT", + "MAIL_SSL", + "MAIL_TLS", "MAIL_USER", "MEMDEBUG_ENABLED", "MEMDEBUG_NOTIFY", @@ -149,10 +170,12 @@ __all__ = [ "REDIRECT_MAX_TIMES", "REDIRECT_PRIORITY_ADJUST", "REFERER_ENABLED", + "REFERRER_POLICIES", "REFERRER_POLICY", "REQUEST_FINGERPRINTER_CLASS", "RETRY_ENABLED", "RETRY_EXCEPTIONS", + "RETRY_GIVE_UP_LOG_LEVEL", "RETRY_HTTP_CODES", "RETRY_PRIORITY_ADJUST", "RETRY_TIMES", @@ -193,9 +216,6 @@ __all__ = [ ADDONS = {} -AJAXCRAWL_ENABLED = False -AJAXCRAWL_MAXSIZE = 32768 - ASYNCIO_EVENT_LOOP = None AUTOTHROTTLE_ENABLED = False @@ -204,12 +224,22 @@ AUTOTHROTTLE_MAX_DELAY = 60.0 AUTOTHROTTLE_START_DELAY = 5.0 AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 +AWS_ACCESS_KEY_ID = None +AWS_SECRET_ACCESS_KEY = None +AWS_ENDPOINT_URL = None +AWS_REGION_NAME = None +AWS_SESSION_TOKEN = None +AWS_USE_SSL = None +AWS_VERIFY = None + BOT_NAME = "scrapybot" CLOSESPIDER_ERRORCOUNT = 0 CLOSESPIDER_ITEMCOUNT = 0 CLOSESPIDER_PAGECOUNT = 0 CLOSESPIDER_TIMEOUT = 0 +CLOSESPIDER_PAGECOUNT_NO_ITEM = 0 +CLOSESPIDER_TIMEOUT_NO_ITEM = 0 COMMANDS_MODULE = "" @@ -262,15 +292,19 @@ DOWNLOAD_HANDLERS_BASE = { DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m +DOWNLOAD_SLOTS = {} + DOWNLOAD_TIMEOUT = 180 # 3mins +DOWNLOAD_TLS_MAX_VERSION = None +DOWNLOAD_TLS_MIN_VERSION = None + DOWNLOAD_VERIFY_CERTIFICATES = False DOWNLOADER = "scrapy.core.downloader.Downloader" DOWNLOADER_CLIENTCONTEXTFACTORY = "SENTINEL" DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT" -# Use highest TLS/SSL protocol version supported by the platform, also allowing negotiation: DOWNLOADER_CLIENT_TLS_METHOD = "TLS" DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False @@ -297,6 +331,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { DOWNLOADER_STATS = True DUPEFILTER_CLASS = "scrapy.dupefilters.RFPDupeFilter" +DUPEFILTER_DEBUG = False EDITOR = "vi" if sys.platform == "win32": @@ -347,8 +382,10 @@ FEED_STORAGE_FTP_ACTIVE = False FEED_STORAGE_GCS_ACL = "" FEED_STORAGE_S3_ACL = "" FEED_TEMPDIR = None +FEED_URI = None FEED_URI_PARAMS = None # a function to extend uri arguments +FILES_STORE = None FILES_STORE_GCS_ACL = "" FILES_STORE_S3_ACL = "private" @@ -373,9 +410,15 @@ HTTPCACHE_IGNORE_SCHEMES = ["file"] HTTPCACHE_POLICY = "scrapy.extensions.httpcache.DummyPolicy" HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" +HTTPERROR_ALLOW_ALL = False +HTTPERROR_ALLOWED_CODES = [] + HTTPPROXY_ENABLED = True HTTPPROXY_AUTH_ENCODING = "latin-1" +HTTPX_HTTP2_ENABLED = False + +IMAGES_STORE = None IMAGES_STORE_GCS_ACL = "" IMAGES_STORE_S3_ACL = "private" @@ -416,6 +459,8 @@ MAIL_HOST = "localhost" MAIL_PORT = 25 MAIL_USER = None MAIL_PASS = None +MAIL_SSL = False +MAIL_TLS = False MEMDEBUG_ENABLED = False # enable memory debugging MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown @@ -446,6 +491,7 @@ REDIRECT_PRIORITY_ADJUST = +2 REFERER_ENABLED = True REFERRER_POLICY = "scrapy.spidermiddlewares.referer.DefaultReferrerPolicy" +REFERRER_POLICIES = {} REQUEST_FINGERPRINTER_CLASS = "scrapy.utils.request.RequestFingerprinter" @@ -464,6 +510,7 @@ RETRY_EXCEPTIONS = [ OSError, "scrapy.core.downloader.handlers.http11.TunnelError", ] +RETRY_GIVE_UP_LOG_LEVEL = "ERROR" RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429] RETRY_PRIORITY_ADJUST = -1 RETRY_TIMES = 2 # initial response + 2 retries = 3 requests @@ -532,7 +579,7 @@ USER_AGENT = f"Scrapy/{import_module('scrapy').__version__} (+https://scrapy.org WARN_ON_GENERATOR_RETURN_VALUE = True -def __getattr__(name: str): +def __getattr__(name: str) -> Any: if name == "CONCURRENT_REQUESTS_PER_IP": import warnings # noqa: PLC0415 @@ -545,4 +592,4 @@ def __getattr__(name: str): ) return 0 - raise AttributeError + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/scrapy/shell.py b/scrapy/shell.py index 44e542470..0bd71feb2 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -28,11 +28,7 @@ from scrapy.spiders import Spider from scrapy.utils.conf import get_config from scrapy.utils.console import DEFAULT_PYTHON_SHELLS, start_python_console from scrapy.utils.datatypes import SequenceExclude -from scrapy.utils.defer import ( - _schedule_coro, - deferred_f_from_coro_f, - maybe_deferred_to_future, -) +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.misc import load_object from scrapy.utils.reactor import is_asyncio_reactor_installed, set_asyncio_event_loop from scrapy.utils.response import open_in_browser @@ -69,8 +65,8 @@ if TYPE_CHECKING: # 3. When fetch() is called, it prepares a request and calls Shell._schedule() # in the reactor thread (via threads.blockingCallFromThread()). # 4. Shell._schedule() calls Shell._open_spider() (on the first call). -# 5. Shell._open_spider() calls engine.open_spider_async(close_if_idle=False) -# and engine._start_request_processing(). +# 5. Shell._open_spider() calls +# engine.open_spider_async(close_if_idle=False). # 6. Shell._schedule() calls engine.crawl(request), scheduling the request. # 7. Shell._schedule() via _request_deferred() waits until the request callback # is called. When it's called, the response becomes available. @@ -214,7 +210,6 @@ class Shell: self.crawler.spider = spider assert self.crawler.engine await self.crawler.engine.open_spider_async(close_if_idle=False) - _schedule_coro(self.crawler.engine._start_request_processing()) self.spider = spider def fetch( diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 299a5d43f..6e8c67adb 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -24,7 +24,7 @@ if TYPE_CHECKING: from scrapy.crawler import Crawler from scrapy.http.request import CallbackT - from scrapy.settings import BaseSettings, _SettingsKey + from scrapy.settings import BaseSettings from scrapy.utils.log import SpiderLoggerAdapter @@ -37,7 +37,7 @@ class Spider(object_ref): """ name: str - custom_settings: dict[_SettingsKey, Any] | None = None + custom_settings: dict[str, Any] | None = None #: Start URLs. See :meth:`start`. start_urls: list[str] diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index f0d093c6e..d2da31f35 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -12,6 +12,7 @@ import warnings from collections.abc import AsyncIterator, Awaitable, Callable from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.link import Link from scrapy.linkextractors import LinkExtractor @@ -46,7 +47,9 @@ def _identity_process_request(request: Request, response: Response) -> Request | return request -def _get_method(method: Callable | str | None, spider: Spider) -> Callable | None: +def _get_method( + method: Callable[..., Any] | str | None, spider: Spider +) -> Callable[..., Any] | None: if callable(method): return method if isinstance(method, str): @@ -218,4 +221,11 @@ class CrawlSpider(Spider): def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self: spider = super().from_crawler(crawler, *args, **kwargs) spider._follow_links = crawler.settings.getbool("CRAWLSPIDER_FOLLOW_LINKS") + if not spider._follow_links: + warnings.warn( + "The CRAWLSPIDER_FOLLOW_LINKS setting is deprecated." + " You can set follow=False in your rules to achieve the same effect.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) return spider diff --git a/scrapy/squeues.py b/scrapy/squeues.py index 7007cd4b8..e06e38e23 100644 --- a/scrapy/squeues.py +++ b/scrapy/squeues.py @@ -26,7 +26,7 @@ if TYPE_CHECKING: def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]: class DirectoriesCreated(queue_class): # type: ignore[valid-type,misc] - def __init__(self, path: str | PathLike, *args: Any, **kwargs: Any): + def __init__(self, path: str | PathLike[str], *args: Any, **kwargs: Any): dirname = Path(path).parent if not dirname.exists(): dirname.mkdir(parents=True, exist_ok=True) diff --git a/scrapy/templates/project/module/items.py.tmpl b/scrapy/templates/project/module/items.py.tmpl index 88a18331c..e7d525f36 100644 --- a/scrapy/templates/project/module/items.py.tmpl +++ b/scrapy/templates/project/module/items.py.tmpl @@ -3,10 +3,11 @@ # See documentation in: # https://docs.scrapy.org/en/latest/topics/items.html -import scrapy +from dataclasses import dataclass -class ${ProjectName}Item(scrapy.Item): +@dataclass +class ${ProjectName}Item: # define the fields for your item here like: - # name = scrapy.Field() + # name: str | None = None pass diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py index cb5424476..fad7e6f6b 100644 --- a/scrapy/utils/_deps_compat.py +++ b/scrapy/utils/_deps_compat.py @@ -6,6 +6,8 @@ from twisted.python.versions import Version as TxVersion TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0) # changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506 TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0) +# lowerMaximumSecurityTo off-by-1, https://github.com/twisted/twisted/issues/10232 +TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0) PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING) # SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object diff --git a/scrapy/utils/_download_handlers.py b/scrapy/utils/_download_handlers.py index 9538dd81e..23b715ac6 100644 --- a/scrapy/utils/_download_handlers.py +++ b/scrapy/utils/_download_handlers.py @@ -2,8 +2,8 @@ from __future__ import annotations -from abc import ABC from contextlib import contextmanager +from http.cookiejar import CookieJar from typing import TYPE_CHECKING, Any from twisted.internet.defer import CancelledError @@ -15,7 +15,6 @@ from twisted.web.client import ResponseFailed from twisted.web.error import SchemeNotSupported from scrapy import responsetypes -from scrapy.core.downloader.handlers.base import BaseDownloadHandler from scrapy.exceptions import ( CannotResolveHostError, DownloadCancelledError, @@ -29,29 +28,24 @@ from scrapy.utils.log import logger if TYPE_CHECKING: from collections.abc import Iterator + from http.client import HTTPResponse + from http.cookiejar import Cookie from ipaddress import IPv4Address, IPv6Address - - from twisted.internet.ssl import Certificate + from urllib.request import Request as ULRequest from scrapy import Request from scrapy.crawler import Crawler from scrapy.http import Headers, Response -class BaseHttpDownloadHandler(BaseDownloadHandler, ABC): - """Base class for built-in HTTP download handlers.""" +class NullCookieJar(CookieJar): # pragma: no cover + """A CookieJar that rejects all cookies.""" - def __init__(self, crawler: Crawler): - super().__init__(crawler) - self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE") - self._default_warnsize: int = crawler.settings.getint("DOWNLOAD_WARNSIZE") - self._fail_on_dataloss: bool = crawler.settings.getbool( - "DOWNLOAD_FAIL_ON_DATALOSS" - ) - self._tls_verbose_logging: bool = crawler.settings.getbool( - "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING" - ) - self._fail_on_dataloss_warned: bool = False + def extract_cookies(self, response: HTTPResponse, request: ULRequest) -> None: + pass + + def set_cookie(self, cookie: Cookie) -> None: + pass @contextmanager @@ -103,7 +97,7 @@ def make_response( headers: Headers, body: bytes = b"", flags: list[str] | None = None, - certificate: Certificate | None = None, + certificate: Any = None, ip_address: IPv4Address | IPv6Address | None = None, protocol: str | None = None, stop_download: StopDownload | None = None, diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index ecb4c1492..44604c0fe 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -148,7 +148,7 @@ class AsyncioLoopingCall: self._func: Callable[_P, _T] = func self._args: tuple[Any, ...] = args self._kwargs: dict[str, Any] = kwargs - self._task: asyncio.Task | None = None + self._task: asyncio.Task[None] | None = None self.interval: float | None = None self._start_time: float | None = None diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py index 5869cf52e..4850b370b 100644 --- a/scrapy/utils/conf.py +++ b/scrapy/utils/conf.py @@ -71,8 +71,8 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]: def closest_scrapy_cfg( - path: str | os.PathLike = ".", - prevpath: str | os.PathLike | None = None, + path: str | os.PathLike[str] = ".", + prevpath: str | os.PathLike[str] | None = None, ) -> str: """Return the path to the closest scrapy.cfg file by traversing the current directory and its parents diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 146e3ae56..4e65c062e 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -28,7 +28,7 @@ _KT = TypeVar("_KT") _VT = TypeVar("_VT") -class CaselessDict(dict): +class CaselessDict(dict): # type: ignore[type-arg] __slots__ = () def __new__(cls, *args: Any, **kwargs: Any) -> Self: @@ -99,20 +99,20 @@ class CaselessDict(dict): return dict.pop(self, self.normkey(key), *args) -class CaseInsensitiveDict(collections.UserDict): +class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]): """A dict-like structure that accepts strings or bytes as keys and allows case-insensitive lookups. """ def __init__(self, *args: Any, **kwargs: Any) -> None: - self._keys: dict = {} + self._keys: dict[str | bytes, Any] = {} super().__init__(*args, **kwargs) - def __getitem__(self, key: AnyStr) -> Any: + def __getitem__(self, key: str | bytes) -> Any: normalized_key = self._normkey(key) return super().__getitem__(self._keys[normalized_key.lower()]) - def __setitem__(self, key: AnyStr, value: Any) -> None: + def __setitem__(self, key: str | bytes, value: Any) -> None: normalized_key = self._normkey(key) try: lower_key = self._keys[normalized_key.lower()] @@ -122,19 +122,19 @@ class CaseInsensitiveDict(collections.UserDict): super().__setitem__(normalized_key, self._normvalue(value)) self._keys[normalized_key.lower()] = normalized_key - def __delitem__(self, key: AnyStr) -> None: + def __delitem__(self, key: str | bytes) -> None: normalized_key = self._normkey(key) stored_key = self._keys.pop(normalized_key.lower()) super().__delitem__(stored_key) - def __contains__(self, key: AnyStr) -> bool: # type: ignore[override] + def __contains__(self, key: str | bytes) -> bool: # type: ignore[override] normalized_key = self._normkey(key) return normalized_key.lower() in self._keys def __repr__(self) -> str: return f"<{self.__class__.__name__}: {super().__repr__()}>" - def _normkey(self, key: AnyStr) -> AnyStr: + def _normkey(self, key: str | bytes) -> str | bytes: return key def _normvalue(self, value: Any) -> Any: @@ -158,7 +158,7 @@ class LocalCache(OrderedDict[_KT, _VT]): super().__setitem__(key, value) -class LocalWeakReferencedCache(weakref.WeakKeyDictionary): +class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]): """ A weakref.WeakKeyDictionary implementation that uses LocalCache as its underlying data structure, making it ordered and capable of being size-limited. @@ -172,9 +172,9 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary): def __init__(self, limit: int | None = None): super().__init__() - self.data: LocalCache = LocalCache(limit=limit) + self.data: LocalCache[_KT, _VT] = LocalCache(limit=limit) - def __setitem__(self, key: _KT, value: _VT) -> None: + def __setitem__(self, key: _KT, value: _VT | None) -> None: # if raised, key is not weak-referenceable, skip caching with contextlib.suppress(TypeError): super().__setitem__(key, value) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 90fd04777..29a34d4ef 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -173,7 +173,7 @@ def parallel( return DeferredList([coop.coiterate(work) for _ in range(count)]) -class _AsyncCooperatorAdapter(Iterator, Generic[_T]): +class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_T]): """A class that wraps an async iterable into a normal iterator suitable for using in Cooperator.coiterate(). As it's only needed for parallel_async(), it calls the callable directly in the callback, instead of providing a more diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py index 102362506..09b67805d 100644 --- a/scrapy/utils/log.py +++ b/scrapy/utils/log.py @@ -12,7 +12,7 @@ from twisted.python import log as twisted_log from twisted.python.failure import Failure import scrapy -from scrapy.settings import Settings, _SettingsKey +from scrapy.settings import Settings from scrapy.utils.versions import get_versions if TYPE_CHECKING: @@ -89,7 +89,7 @@ DEFAULT_LOGGING = { def configure_logging( - settings: Settings | dict[_SettingsKey, Any] | None = None, + settings: Settings | dict[str, Any] | None = None, install_root_handler: bool = True, ) -> None: """ @@ -260,7 +260,8 @@ def logformatter_adapter( return (level, message, args) -class SpiderLoggerAdapter(logging.LoggerAdapter): +# LoggerAdapter is only parameterized since Python 3.11 +class SpiderLoggerAdapter(logging.LoggerAdapter): # type: ignore[type-arg] def process( self, msg: str, kwargs: MutableMapping[str, Any] ) -> tuple[str, MutableMapping[str, Any]]: diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 3baa02e2b..0b67eaa34 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -28,7 +28,7 @@ if TYPE_CHECKING: _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes -_ITER_T = TypeVar("_ITER_T", bound=dict | Item | str | bytes) +_ITER_T = TypeVar("_ITER_T", bound=dict[Any, Any] | Item | str | bytes) _T = TypeVar("_T") _T_co = TypeVar("_T_co", covariant=True) _P = ParamSpec("_P") @@ -252,7 +252,9 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]: yield node -_generator_callbacks_cache = LocalWeakReferencedCache(limit=128) +_generator_callbacks_cache: LocalWeakReferencedCache[Callable[..., Any], bool] = ( + LocalWeakReferencedCache(limit=128) +) def _returns_none(return_node: ast.Return) -> bool: diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 9cb695111..00cf818b7 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -8,12 +8,14 @@ import gc import inspect import re import sys +import warnings import weakref from collections.abc import AsyncIterator, Iterable, Mapping from functools import partial, wraps from itertools import chain from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.asyncgen import as_async_generator if TYPE_CHECKING: @@ -294,12 +296,13 @@ else: gc.collect() -class MutableChain(Iterable[_T]): - """ - Thin wrapper around itertools.chain, allowing to add iterables "in-place" - """ - +class MutableChain(Iterable[_T]): # pragma: no cover def __init__(self, *args: Iterable[_T]): + warnings.warn( + "MutableChain is deprecated and will be removed in a future Scrapy version.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) self.data: Iterator[_T] = chain.from_iterable(args) def extend(self, *iterables: Iterable[_T]) -> None: @@ -356,3 +359,13 @@ def _looks_like_import_path(value: str) -> bool: if any(part == "" for part in parts): return False return all(part.isidentifier() for part in parts) + + +def _iter_exc_causes(exc: BaseException) -> Iterable[BaseException]: + """Iterate over the exception causes/contexts.""" + seen: set[int] = set() + cur: BaseException | None = exc + while cur is not None and id(cur) not in seen: + seen.add(id(cur)) + yield cur + cur = cur.__cause__ or cur.__context__ diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 37db6c25f..7ab58093a 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -55,7 +55,7 @@ class CallLaterOnce(Generic[_T]): self._a: tuple[Any, ...] = a self._kw: dict[str, Any] = kw self._call: CallLaterResult | None = None - self._deferreds: list[Deferred] = [] + self._deferreds: list[Deferred[None]] = [] def schedule(self, delay: float = 0) -> None: # circular import diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index 27d669e71..ffb7fae49 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -28,7 +28,7 @@ if TYPE_CHECKING: _fingerprint_cache: WeakKeyDictionary[ - Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes] + Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes] ] = WeakKeyDictionary() @@ -71,8 +71,10 @@ def fingerprint( processed_include_headers = tuple( to_bytes(h.lower()) for h in sorted(include_headers) ) + verbatim_url = bool(request.meta.get("verbatim_url")) + effective_keep_fragments = keep_fragments and not verbatim_url cache = _fingerprint_cache.setdefault(request, {}) - cache_key = (processed_include_headers, keep_fragments) + cache_key = (processed_include_headers, effective_keep_fragments, verbatim_url) if cache_key not in cache: # To decode bytes reliably (JSON does not support bytes), regardless of # character encoding, we use bytes.hex() @@ -84,9 +86,13 @@ def fingerprint( header_value.hex() for header_value in request.headers.getlist(header) ] + if verbatim_url: + url = request.url + else: + url = canonicalize_url(request.url, keep_fragments=keep_fragments) fingerprint_data = { "method": to_unicode(request.method), - "url": canonicalize_url(request.url, keep_fragments=keep_fragments), + "url": url, "body": (request.body or b"").hex(), "headers": headers, } @@ -108,8 +114,9 @@ class RequestFingerprinter: (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url ` and the values of :attr:`request.method ` and :attr:`request.body - `. It then generates an `SHA1 - `_ hash. + `, unless :reqmeta:`verbatim_url` is true for that + request. It then generates an `SHA1 `_ + hash. """ @classmethod diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index abb5f6a70..c068d9b1e 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -97,8 +97,7 @@ def open_in_browser( # XXX: this implementation is a bit dirty and could be improved body = response.body if isinstance(response, HtmlResponse): - if b"' body = re.sub(rb"]*?>)", to_bytes(repl), body, count=1) ext = ".html" diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index d9a72273a..ee391a49e 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -186,7 +186,9 @@ async def _send_catch_log_asyncio( handlers: list[Awaitable[TypingAny]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - async def handler(receiver: Callable) -> tuple[Callable, TypingAny]: + async def handler( + receiver: Callable[..., Any], + ) -> tuple[Callable[..., Any], TypingAny]: result: TypingAny try: result = await ensure_awaitable( diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index a8b9cd225..22e9414b8 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -2,30 +2,59 @@ from __future__ import annotations import logging import ssl -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TypedDict, TypeVar import OpenSSL._util as pyOpenSSLutil import OpenSSL.SSL import OpenSSL.version +from twisted.internet.ssl import CertificateOptions, TLSVersion +from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1 from scrapy.utils.python import to_unicode if TYPE_CHECKING: + from collections.abc import Callable + from OpenSSL.crypto import X509Name from scrapy.settings import BaseSettings logger = logging.getLogger(__name__) +_T = TypeVar("_T") + + +# common + + +def _get_tls_version_limit( + settings: BaseSettings, setting_name: str, converter: Callable[[str], _T] +) -> _T | None: + setting: str | None = settings[setting_name] + if setting is None: + return None + try: + return converter(setting) + except Exception as ex: + raise ValueError(f"Unknown {setting_name} value: {setting}") from ex + + +def _get_tls_version_limits( + settings: BaseSettings, converter: Callable[[str], _T] +) -> tuple[_T | None, _T | None]: + return ( + _get_tls_version_limit(settings, "DOWNLOAD_TLS_MIN_VERSION", converter), + _get_tls_version_limit(settings, "DOWNLOAD_TLS_MAX_VERSION", converter), + ) + # stdlib ssl module utils -# possible documented values for DOWNLOADER_CLIENT_TLS_METHOD -_STDLIB_PROTOCOL_MAP = { - "TLS": ssl.PROTOCOL_TLS_CLIENT, - "TLSv1.0": ssl.PROTOCOL_TLSv1, - "TLSv1.1": ssl.PROTOCOL_TLSv1_1, - "TLSv1.2": ssl.PROTOCOL_TLSv1_2, +_STDLIB_VERSION_MAP: dict[str, ssl.TLSVersion] = { + "TLSv1.0": ssl.TLSVersion.TLSv1, + "TLSv1.1": ssl.TLSVersion.TLSv1_1, + "TLSv1.2": ssl.TLSVersion.TLSv1_2, + "TLSv1.3": ssl.TLSVersion.TLSv1_3, } @@ -35,13 +64,13 @@ def _make_ssl_context(settings: BaseSettings) -> ssl.SSLContext: It's intended to be used in an HTTPS download handler. """ - method_setting: str = settings["DOWNLOADER_CLIENT_TLS_METHOD"] - if method_setting not in _STDLIB_PROTOCOL_MAP: - raise ValueError(f"Unsupported TLS method: {method_setting}") + tls_min_ver, tls_max_ver = _get_tls_version_limits( + settings, _STDLIB_VERSION_MAP.__getitem__ + ) ciphers_setting: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"] verify_setting = settings.getbool("DOWNLOAD_VERIFY_CERTIFICATES") - ctx = ssl.SSLContext(_STDLIB_PROTOCOL_MAP[method_setting]) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) if verify_setting: ctx.check_hostname = True ctx.verify_mode = ssl.CERT_REQUIRED @@ -49,11 +78,27 @@ def _make_ssl_context(settings: BaseSettings) -> ssl.SSLContext: else: ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE + if tls_min_ver is not None: + ctx.minimum_version = tls_min_ver + if tls_max_ver is not None: + ctx.maximum_version = tls_max_ver if ciphers_setting: ctx.set_ciphers(ciphers_setting) return ctx +def _make_insecure_ssl_ctx() -> ssl.SSLContext: + """Create an SSL context that doesn't verify certificates. + + Compared to :func:`~scrapy.utils.ssl._make_ssl_context` this is much more + simple. + """ + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + return ctx + + def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None: cipher = sslobj.cipher() logger.debug( @@ -61,8 +106,11 @@ def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None: f" using protocol {sslobj.version()}," f" cipher {cipher[0] if cipher else None}" ) - # The peer certificate is unavailable on SSLObject unless peer - # certificate verification is enabled, which we don't want. + if cert := sslobj.getpeercert(): + # Not available without certificate verification + logger.debug( + f'SSL connection certificate: issuer "{cert["issuer"]}", subject "{cert["subject"]}"' + ) # pyOpenSSL utils @@ -139,3 +187,42 @@ def _log_ssl_conn_debug_info(hostname: str, connection: OpenSSL.SSL.Connection) key_info = get_temp_key_info(connection._ssl) if key_info: logger.debug("SSL temp key: %s", key_info) + + +# Twisted-specific + + +class _CertificateOptionsVersionKwargs(TypedDict, total=False): + lowerMaximumSecurityTo: TLSVersion + insecurelyLowerMinimumTo: TLSVersion + raiseMinimumTo: TLSVersion + + +def _get_cert_options_version_kwargs( + min_version: TLSVersion | None, max_version: TLSVersion | None +) -> _CertificateOptionsVersionKwargs: + """Get TLS version kwargs for + :class:`~twisted.internet.ssl.CertificateOptions` for the given limits.""" + result: _CertificateOptionsVersionKwargs = {} + if max_version: + if TWISTED_TLS_LIMITS_OFFBY1: + # lowerMaximumSecurityTo is treated as 1 version lower than the passed one + versions = list(TLSVersion.iterconstants()) + max_index = versions.index(max_version) + if max_index + 1 >= len(versions): + raise ValueError( + f"Due to an error in Twisted < 26.4.0 cannot set the maximum TLS version to {max_version.name}" + ) + max_version = versions[max_index + 1] + result["lowerMaximumSecurityTo"] = max_version + if min_version: + # We cannot pass both insecurelyLowerMinimumTo and raiseMinimumTo, + # so we need to know the direction. + + # 1.0 in Twisted 22.8.0 and older, 1.2 in Twisted 22.10.0 and newer + default_min = CertificateOptions._defaultMinimumTLSVersion + if min_version < default_min: + result["insecurelyLowerMinimumTo"] = min_version + elif min_version > default_min: + result["raiseMinimumTo"] = min_version + return result diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py index 3e4dae5c8..977d5a42a 100644 --- a/scrapy/utils/template.py +++ b/scrapy/utils/template.py @@ -11,7 +11,7 @@ if TYPE_CHECKING: from os import PathLike -def render_templatefile(path: str | PathLike, **kwargs: Any) -> None: +def render_templatefile(path: str | PathLike[str], **kwargs: Any) -> None: path_obj = Path(path) raw = path_obj.read_text("utf8") diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py index 082aca4b1..87df10a02 100644 --- a/scrapy/utils/trackref.py +++ b/scrapy/utils/trackref.py @@ -30,7 +30,9 @@ if TYPE_CHECKING: from typing_extensions import Self -live_refs: defaultdict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary) +live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict( + WeakKeyDictionary +) class object_ref: diff --git a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py index 93259f696..f49741441 100644 --- a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py +++ b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py @@ -23,7 +23,7 @@ class NoRequestsSpider(scrapy.Spider): yield -def log_task_exception(task: Task) -> None: +def log_task_exception(task: Task[None]) -> None: try: task.result() except Exception: diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py index f6fead718..b85f60f87 100644 --- a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py +++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py @@ -17,7 +17,7 @@ class AsyncioReactorSpider(scrapy.Spider): } -def log_task_exception(task: Task) -> None: +def log_task_exception(task: Task[None]) -> None: try: task.result() except Exception: diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py index e358eaca4..98fe58860 100644 --- a/tests/mocks/dummydbm.py +++ b/tests/mocks/dummydbm.py @@ -4,7 +4,7 @@ from collections import defaultdict from typing import Any -class DummyDB(dict): +class DummyDB(dict): # type: ignore[type-arg] """Provide dummy DBM-like interface.""" def close(self): diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py index 622324a10..7ad873c02 100644 --- a/tests/mockserver/http.py +++ b/tests/mockserver/http.py @@ -34,6 +34,7 @@ from .http_resources import ( ResponseHeadersResource, SetCookie, Status, + UriResource, ) @@ -84,6 +85,7 @@ class Root(resource.Resource): self.putChild(b"duplicate-header", DuplicateHeaderResource()) self.putChild(b"response-headers", ResponseHeadersResource()) self.putChild(b"set-cookie", SetCookie()) + self.putChild(b"uri", UriResource()) def getChild(self, path, request): return self diff --git a/tests/mockserver/http_base.py b/tests/mockserver/http_base.py index ffd11c2dd..7b38409ff 100644 --- a/tests/mockserver/http_base.py +++ b/tests/mockserver/http_base.py @@ -106,6 +106,16 @@ def main_factory( default=None, help="SSL cipher string (optional)", ) + parser.add_argument( + "--tls-min-version", + default=None, + help="Minimum accepted TLS version (optional)", + ) + parser.add_argument( + "--tls-max-version", + default=None, + help="Maximum accepted TLS version (optional)", + ) args = parser.parse_args() context_factory_kw = {} if args.keyfile: @@ -114,6 +124,10 @@ def main_factory( context_factory_kw["certfile"] = args.certfile if args.cipher_string: context_factory_kw["cipher_string"] = args.cipher_string + if args.tls_min_version: + context_factory_kw["tls_min_version"] = args.tls_min_version + if args.tls_max_version: + context_factory_kw["tls_max_version"] = args.tls_max_version context_factory = ssl_context_factory(**context_factory_kw) https_port = reactor.listenSSL(0, factory, context_factory) diff --git a/tests/mockserver/mitm_proxy.py b/tests/mockserver/mitm_proxy.py index 1ec5a79b5..56620f84e 100644 --- a/tests/mockserver/mitm_proxy.py +++ b/tests/mockserver/mitm_proxy.py @@ -11,6 +11,9 @@ class MitmProxy: auth_user = "scrapy" auth_pass = "scrapy" + def __init__(self, mode: str | None = None) -> None: + self.mode = mode + def start(self) -> str: script = """ import sys @@ -32,6 +35,8 @@ sys.exit(mitmdump()) "-s", str(Path(__file__).with_name("mitm_proxy_addon.py")), ] + if self.mode: + args += ["--mode", self.mode] self.proc: Popen[str] = Popen( [ sys.executable, @@ -44,12 +49,13 @@ sys.exit(mitmdump()) text=True, ) assert self.proc.stdout is not None + scheme = "socks5" if self.mode == "socks5" else "http" line = "" for line in self.proc.stdout: - m = re.search(r"listening at (?:http://)?([^:]+:\d+)", line) + m = re.search(r"listening at (?:\w+://)?([^:]+:\d+)", line) if m: host_port = m.group(1) - return f"http://{self.auth_user}:{self.auth_pass}@{host_port}" + return f"{scheme}://{self.auth_user}:{self.auth_pass}@{host_port}" self.stop() raise RuntimeError(f"Failed to parse mitmdump output: {line}") diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.py index a8f483ee1..943775fa5 100644 --- a/tests/mockserver/simple_https.py +++ b/tests/mockserver/simple_https.py @@ -21,11 +21,21 @@ class SimpleMockServer(BaseMockServer): listen_http = False module_name = "tests.mockserver.simple_https" - def __init__(self, keyfile: str, certfile: str, cipher_string: str | None): + def __init__( + self, + keyfile: str, + certfile: str, + *, + cipher_string: str | None = None, + tls_min_version: str | None = None, + tls_max_version: str | None = None, + ): super().__init__() self.keyfile = keyfile self.certfile = certfile self.cipher_string = cipher_string or "" + self.tls_min_version = tls_min_version + self.tls_max_version = tls_max_version def get_additional_args(self) -> list[str]: args = [ @@ -36,6 +46,10 @@ class SimpleMockServer(BaseMockServer): ] if self.cipher_string is not None: args.extend(["--cipher-string", self.cipher_string]) + if self.tls_min_version is not None: + args.extend(["--tls-min-version", self.tls_min_version]) + if self.tls_max_version is not None: + args.extend(["--tls-max-version", self.tls_max_version]) return args diff --git a/tests/mockserver/utils.py b/tests/mockserver/utils.py index 17d78ecdd..7aa656780 100644 --- a/tests/mockserver/utils.py +++ b/tests/mockserver/utils.py @@ -9,8 +9,10 @@ from OpenSSL import SSL from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey from twisted.internet.ssl import CertificateOptions, ContextFactory +from scrapy.core.downloader.tls import _TWISTED_VERSION_MAP from scrapy.utils._deps_compat import PYOPENSSL_WANTS_X509_PKEY from scrapy.utils.python import to_bytes +from scrapy.utils.ssl import _get_cert_options_version_kwargs if TYPE_CHECKING: from twisted.internet.interfaces import IOpenSSLContextFactory @@ -19,7 +21,10 @@ if TYPE_CHECKING: def ssl_context_factory( keyfile: str = "keys/localhost.key", certfile: str = "keys/localhost.crt", + *, cipher_string: str | None = None, + tls_min_version: str | None = None, + tls_max_version: str | None = None, ) -> IOpenSSLContextFactory: keyfile_path = Path(__file__).parent.parent / keyfile certfile_path = Path(__file__).parent.parent / certfile @@ -31,10 +36,14 @@ def ssl_context_factory( cert = load_certificate(FILETYPE_PEM, certfile_path.read_bytes()) # type: ignore[assignment] key = load_privatekey(FILETYPE_PEM, keyfile_path.read_bytes()) # type: ignore[assignment] + tls_min = _TWISTED_VERSION_MAP.get(tls_min_version) if tls_min_version else None + tls_max = _TWISTED_VERSION_MAP.get(tls_max_version) if tls_max_version else None + tls_version_kwargs = _get_cert_options_version_kwargs(tls_min, tls_max) # https://github.com/twisted/twisted/issues/12638 factory: CertificateOptions = CertificateOptions( privateKey=key, # type: ignore[arg-type] certificate=cert, # type: ignore[arg-type] + **tls_version_kwargs, ) if cipher_string: ctx = factory.getContext() diff --git a/tests/spiders.py b/tests/spiders.py index 373dc00bd..55d1ea365 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -94,19 +94,19 @@ class DelaySpider(MetaSpider): class LogSpider(MetaSpider): name = "log_spider" - def log_debug(self, message: str, extra: dict | None = None): + def log_debug(self, message: str, extra: dict[str, Any] | None = None): self.logger.debug(message, extra=extra) - def log_info(self, message: str, extra: dict | None = None): + def log_info(self, message: str, extra: dict[str, Any] | None = None): self.logger.info(message, extra=extra) - def log_warning(self, message: str, extra: dict | None = None): + def log_warning(self, message: str, extra: dict[str, Any] | None = None): self.logger.warning(message, extra=extra) - def log_error(self, message: str, extra: dict | None = None): + def log_error(self, message: str, extra: dict[str, Any] | None = None): self.logger.error(message, extra=extra) - def log_critical(self, message: str, extra: dict | None = None): + def log_critical(self, message: str, extra: dict[str, Any] | None = None): self.logger.critical(message, extra=extra) def parse(self, response): @@ -417,7 +417,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider): """ name = "crawl_spider_with_parse_method" - custom_settings: dict = { + custom_settings: dict[str, Any] = { "RETRY_HTTP_CODES": [], # no need to retry } rules = (Rule(LinkExtractor(), callback="parse", follow=True),) diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py index fd9505060..67e3eb50a 100644 --- a/tests/test_command_genspider.py +++ b/tests/test_command_genspider.py @@ -9,7 +9,7 @@ from tests.test_commands import TestProjectBase from tests.utils.cmdline import call, proc -def find_in_file(filename: Path, regex: str) -> re.Match | None: +def find_in_file(filename: Path, regex: str) -> re.Match[str] | None: """Find first pattern occurrence in file""" pattern = re.compile(regex) with filename.open("r", encoding="utf-8") as f: diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index b407211dd..1585835cc 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -4,7 +4,7 @@ import os import sys from io import BytesIO from pathlib import Path -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING import pytest from pexpect.popen_spawn import PopenSpawn @@ -151,8 +151,7 @@ class TestInteractiveShell: env = os.environ.copy() env["SCRAPY_PYTHON_SHELL"] = "python" logfile = BytesIO() - # https://github.com/python/typeshed/issues/14915 - p = PopenSpawn(args, env=cast("os._Environ", env), timeout=5) + p = PopenSpawn(args, env=env, timeout=5) p.logfile_read = logfile p.expect_exact("Available Scrapy objects") p.sendline(f"fetch('{mockserver.url('/')}')") diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py index 4eed09fcd..2a9d0ed57 100644 --- a/tests/test_command_startproject.py +++ b/tests/test_command_startproject.py @@ -79,7 +79,7 @@ class TestStartprojectCommand: def get_permissions_dict( - path: str | os.PathLike, renamings=None, ignore=None + path: str | os.PathLike[str], renamings=None, ignore=None ) -> dict[str, str]: def get_permissions(path: Path) -> str: return oct(path.stat().st_mode) diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index 43edf1774..abeaa2f65 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -14,7 +14,7 @@ from twisted.web import server, static from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody from twisted.web.client import Response as TxResponse -from scrapy.core.downloader import Downloader, Slot +from scrapy.core.downloader import Downloader, Slot, tls from scrapy.core.downloader.contextfactory import ( _load_context_factory_from_settings, _ScrapyClientContextFactory, @@ -202,18 +202,37 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): def test_setting_none(self): crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": None}) - with pytest.raises(KeyError): + with ( + pytest.warns( + ScrapyDeprecationWarning, + match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated", + ), + pytest.raises(KeyError), + ): _load_context_factory_from_settings(crawler) def test_setting_bad(self): crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "bad"}) - with pytest.raises(KeyError): + with ( + pytest.warns( + ScrapyDeprecationWarning, + match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated", + ), + pytest.raises(KeyError), + ): _load_context_factory_from_settings(crawler) + @pytest.mark.filterwarnings( + r"ignore:Passing method to twisted\.internet\.ssl\.CertificateOptions:DeprecationWarning" + ) @coroutine_test async def test_setting_explicit(self, server_url: str) -> None: crawler = get_crawler(settings_dict={"DOWNLOADER_CLIENT_TLS_METHOD": "TLSv1.2"}) - client_context_factory = _load_context_factory_from_settings(crawler) + with pytest.warns( + ScrapyDeprecationWarning, + match="Setting DOWNLOADER_CLIENT_TLS_METHOD to a non-default value is deprecated", + ): + client_context_factory = _load_context_factory_from_settings(crawler) assert client_context_factory._ssl_method == OpenSSL.SSL.TLSv1_2_METHOD await self._assert_factory_works(server_url, client_context_factory) @@ -227,6 +246,9 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase): assert client_context_factory._ssl_method == OpenSSL.SSL.SSLv23_METHOD await self._assert_factory_works(server_url, client_context_factory) + @pytest.mark.filterwarnings( + r"ignore:Passing method to twisted\.internet\.ssl\.CertificateOptions:DeprecationWarning" + ) @coroutine_test async def test_direct_init(self, server_url: str) -> None: client_context_factory = _ScrapyClientContextFactory(OpenSSL.SSL.TLSv1_2_METHOD) @@ -246,3 +268,16 @@ async def test_fetch_deprecated_spider_arg(): match=r"The fetch\(\) method of .+\.CustomDownloader requires a spider argument", ): await crawler.crawl_async() + + +def test_deprecated_tls_module_names() -> None: + with pytest.warns( + ScrapyDeprecationWarning, + match="scrapy.core.downloader.tls.METHOD_TLS is deprecated", + ): + assert tls.METHOD_TLS == "TLS" + with pytest.warns( + ScrapyDeprecationWarning, + match="scrapy.core.downloader.tls.openssl_methods is deprecated", + ): + assert isinstance(tls.openssl_methods, dict) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 811ce1d18..ada4c31ce 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any from urllib.parse import urlencode, urlparse import pytest -from testfixtures import LogCapture +from cryptography.x509 import load_der_x509_certificate from twisted.internet.defer import succeed from twisted.internet.ssl import Certificate from twisted.python.failure import Failure @@ -23,7 +23,6 @@ from scrapy.utils.engine import format_engine_status, get_engine_status from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler, get_reactor_settings from tests import NON_EXISTING_RESOLVABLE -from tests.mockserver.http import MockServer from tests.spiders import ( AsyncDefAsyncioGenComplexSpider, AsyncDefAsyncioGenExcSpider, @@ -55,44 +54,36 @@ from tests.spiders import ( StartGoodAndBadOutput, StartItemSpider, ) -from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.decorators import coroutine_test if TYPE_CHECKING: from scrapy.statscollectors import StatsCollector + from tests.mockserver.http import MockServer class TestCrawl: - mockserver: MockServer - - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - @inline_callbacks_test - def test_follow_all(self): + @coroutine_test + async def test_follow_all(self, mockserver: MockServer) -> None: crawler = get_crawler(FollowAllSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, FollowAllSpider) assert len(crawler.spider.urls_visited) == 11 # 10 + start_url @coroutine_test - async def test_fixed_delay(self): - await self._test_delay(total=3, delay=0.2) + async def test_fixed_delay(self, mockserver: MockServer) -> None: + await self._test_delay(mockserver, total=3, delay=0.2) @coroutine_test - async def test_randomized_delay(self): - await self._test_delay(total=3, delay=0.1, randomize=True) + async def test_randomized_delay(self, mockserver: MockServer) -> None: + await self._test_delay(mockserver, total=3, delay=0.1, randomize=True) + @staticmethod async def _test_delay( - self, total: int, delay: float, randomize: bool = False + mockserver: MockServer, total: int, delay: float, randomize: bool = False ) -> None: crawl_kwargs = { "maxlatency": delay * 2, - "mockserver": self.mockserver, + "mockserver": mockserver, "total": total, } tolerance = 1 - (0.6 if randomize else 0.2) @@ -121,18 +112,20 @@ class TestCrawl: average = total_time / (len(times) - 1) assert average <= delay / tolerance, "test total or delay values are too small" - @inline_callbacks_test - def test_timeout_success(self): + @coroutine_test + async def test_timeout_success(self, mockserver: MockServer) -> None: crawler = get_crawler(DelaySpider) - yield crawler.crawl(n=0.5, mockserver=self.mockserver) + await crawler.crawl_async(n=0.5, mockserver=mockserver) + assert isinstance(crawler.spider, DelaySpider) assert crawler.spider.t1 > 0 assert crawler.spider.t2 > 0 assert crawler.spider.t2 > crawler.spider.t1 - @inline_callbacks_test - def test_timeout_failure(self): + @coroutine_test + async def test_timeout_failure(self, mockserver: MockServer) -> None: crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) - yield crawler.crawl(n=0.5, mockserver=self.mockserver) + await crawler.crawl_async(n=0.5, mockserver=mockserver) + assert isinstance(crawler.spider, DelaySpider) assert crawler.spider.t1 > 0 assert crawler.spider.t2 == 0 assert crawler.spider.t2_err > 0 @@ -140,81 +133,96 @@ class TestCrawl: # server hangs after receiving response headers crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35}) - yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver) + await crawler.crawl_async(n=0.5, b=1, mockserver=mockserver) + assert isinstance(crawler.spider, DelaySpider) assert crawler.spider.t1 > 0 assert crawler.spider.t2 == 0 assert crawler.spider.t2_err > 0 assert crawler.spider.t2_err > crawler.spider.t1 - @inline_callbacks_test - def test_retry_503(self): + @coroutine_test + async def test_retry_503( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=503"), mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/status?n=503"), mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @inline_callbacks_test - def test_retry_conn_failed(self): + @coroutine_test + async def test_retry_conn_failed( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - "http://localhost:65432/status?n=503", mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + "http://localhost:65432/status?n=503", mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @inline_callbacks_test - def test_retry_dns_error(self): + @coroutine_test + async def test_retry_dns_error( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: if NON_EXISTING_RESOLVABLE: pytest.skip("Non-existing hosts are resolvable") crawler = get_crawler(SimpleSpider) - with LogCapture() as log: + with caplog.at_level(logging.DEBUG): # try to fetch the homepage of a nonexistent domain - yield crawler.crawl( - "http://dns.resolution.invalid./", mockserver=self.mockserver + await crawler.crawl_async( + "http://dns.resolution.invalid./", mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @inline_callbacks_test - def test_start_bug_before_yield(self): - with LogCapture("scrapy", level=logging.ERROR) as log: + @coroutine_test + async def test_start_bug_before_yield( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.ERROR): crawler = get_crawler(BrokenStartSpider) - yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) + await crawler.crawl_async(fail_before_yield=1, mockserver=mockserver) - assert len(log.records) == 1 - record = log.records[0] + assert len(caplog.records) == 1 + record = caplog.records[0] assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @inline_callbacks_test - def test_start_bug_yielding(self): - with LogCapture("scrapy", level=logging.ERROR) as log: + @coroutine_test + async def test_start_bug_yielding( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.ERROR): crawler = get_crawler(BrokenStartSpider) - yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) + await crawler.crawl_async(fail_yielding=1, mockserver=mockserver) - assert len(log.records) == 1 - record = log.records[0] + assert len(caplog.records) == 1 + record = caplog.records[0] assert record.exc_info is not None assert record.exc_info[0] is ZeroDivisionError - @inline_callbacks_test - def test_start_items(self): + @coroutine_test + async def test_start_items( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: items = [] def _on_item_scraped(item): items.append(item) - with LogCapture("scrapy", level=logging.ERROR) as log: + with caplog.at_level(logging.ERROR): crawler = get_crawler(StartItemSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) - assert len(log.records) == 0 + assert len(caplog.records) == 0 assert items == [{"name": "test item"}] - @inline_callbacks_test - def test_start_unsupported_output(self): + @coroutine_test + async def test_start_unsupported_output( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: """Anything that is not a request is assumed to be an item, avoiding a potentially expensive call to itemadapter.is_item(), and letting instead things fail when ItemAdapter is actually used on the @@ -225,35 +233,39 @@ class TestCrawl: def _on_item_scraped(item): items.append(item) - with LogCapture("scrapy", level=logging.ERROR) as log: + with caplog.at_level(logging.ERROR): crawler = get_crawler(StartGoodAndBadOutput) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) - assert len(log.records) == 0 + assert len(caplog.records) == 0 assert len(items) == 3 assert not any(isinstance(item, Request) for item in items) - @inline_callbacks_test - def test_start_dupes(self): + @coroutine_test + async def test_start_dupes(self, mockserver: MockServer) -> None: settings = {"CONCURRENT_REQUESTS": 1} crawler = get_crawler(DuplicateStartSpider, settings) - yield crawler.crawl( - dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver + await crawler.crawl_async( + dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=mockserver ) + assert isinstance(crawler.spider, DuplicateStartSpider) assert crawler.spider.visited == 6 crawler = get_crawler(DuplicateStartSpider, settings) - yield crawler.crawl( + await crawler.crawl_async( dont_filter=False, distinct_urls=3, dupe_factor=4, - mockserver=self.mockserver, + mockserver=mockserver, ) + assert isinstance(crawler.spider, DuplicateStartSpider) assert crawler.spider.visited == 3 - @inline_callbacks_test - def test_unbounded_response(self): + @coroutine_test + async def test_unbounded_response( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # Completeness of responses without Content-Length or Transfer-Encoding # can not be determined, we treat them as valid but flagged as "partial" query = urlencode( @@ -278,41 +290,45 @@ with multiples lines } ) crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url(f"/raw?{query}"), mockserver=mockserver ) - assert str(log).count("Got response 200") == 1 + assert caplog.text.count("Got response 200") == 1 - @inline_callbacks_test - def test_retry_conn_lost(self): + @coroutine_test + async def test_retry_conn_lost( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # connection lost after receiving data crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/drop?abort=0"), mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) - @inline_callbacks_test - def test_retry_conn_aborted(self): + @coroutine_test + async def test_retry_conn_aborted( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # connection lost before receiving data crawler = get_crawler(SimpleSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/drop?abort=1"), mockserver=mockserver ) - self._assert_retried(log) + self._assert_retried(caplog.text) @staticmethod - def _assert_retried(log: LogCapture | str) -> None: + def _assert_retried(log: str) -> None: assert str(log).count("Retrying") == 2 assert str(log).count("Gave up retrying") == 1 - @inline_callbacks_test - def test_referer_header(self): + @coroutine_test + async def test_referer_header(self, mockserver: MockServer) -> None: """Referer header is set by RefererMiddleware unless it is already set""" - req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1) + req0 = Request(mockserver.url("/echo?headers=1&body=0"), dont_filter=True) req1 = req0.replace() req2 = req0.replace(headers={"Referer": None}) req3 = req0.replace(headers={"Referer": "http://example.com"}) @@ -320,7 +336,8 @@ with multiples lines req1.meta["next"] = req2 req2.meta["next"] = req3 crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl(seed=req0, mockserver=self.mockserver) + await crawler.crawl_async(seed=req0, mockserver=mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) # basic asserts in case of weird communication errors assert "responses" in crawler.spider.meta assert "failures" not in crawler.spider.meta @@ -337,33 +354,35 @@ with multiples lines echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body)) assert echo3["headers"].get("Referer") == ["http://example.com"] - @inline_callbacks_test - def test_engine_status(self): + @coroutine_test + async def test_engine_status(self, mockserver: MockServer) -> None: est = [] def cb(response): est.append(get_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl( - seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver + await crawler.crawl_async( + seed=mockserver.url("/"), callback_func=cb, mockserver=mockserver ) + assert isinstance(crawler.spider, SingleRequestSpider) assert len(est) == 1, est s = dict(est[0]) assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == 1 - @inline_callbacks_test - def test_format_engine_status(self): + @coroutine_test + async def test_format_engine_status(self, mockserver: MockServer) -> None: est = [] def cb(response): est.append(format_engine_status(crawler.engine)) crawler = get_crawler(SingleRequestSpider) - yield crawler.crawl( - seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver + await crawler.crawl_async( + seed=mockserver.url("/"), callback_func=cb, mockserver=mockserver ) + assert isinstance(crawler.spider, SingleRequestSpider) assert len(est) == 1, est est = est[0].split("\n")[2:-2] # remove header & footer # convert to dict @@ -376,8 +395,10 @@ with multiples lines assert s["engine.spider.name"] == crawler.spider.name assert s["len(engine.scraper.slot.active)"] == "1" - @inline_callbacks_test - def test_open_spider_error_on_faulty_pipeline(self): + @coroutine_test + async def test_open_spider_error_on_faulty_pipeline( + self, mockserver: MockServer + ) -> None: settings = { "ITEM_PIPELINES": { "tests.pipelines.ZeroDivisionErrorPipeline": 300, @@ -385,25 +406,48 @@ with multiples lines } crawler = get_crawler(SimpleSpider, settings) with pytest.raises(ZeroDivisionError): - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) assert not crawler.crawling - @inline_callbacks_test - def test_crawlerrunner_accepts_crawler(self): - crawler = get_crawler(SimpleSpider) - runner = CrawlerRunner() - with LogCapture() as log: - yield runner.crawl( - crawler, - self.mockserver.url("/status?n=200"), - mockserver=self.mockserver, + @coroutine_test + async def test_open_spider_error_on_faulty_pipeline_crawl( + self, mockserver: MockServer + ) -> None: + # cover the except block in Crawler.crawl() + settings = { + "ITEM_PIPELINES": { + "tests.pipelines.ZeroDivisionErrorPipeline": 300, + } + } + crawler = get_crawler(SimpleSpider, settings) + with pytest.raises(ZeroDivisionError): + await maybe_deferred_to_future( + crawler.crawl(mockserver.url("/status?n=200"), mockserver=mockserver) ) - assert "Got response 200" in str(log) + assert not crawler.crawling @coroutine_test - async def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture) -> None: + async def test_crawlerrunner_accepts_crawler( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + crawler = get_crawler(SimpleSpider) + runner = CrawlerRunner() + with caplog.at_level(logging.DEBUG): + await maybe_deferred_to_future( + runner.crawl( + crawler, + mockserver.url("/status?n=200"), + mockserver=mockserver, + ) + ) + assert "Got response 200" in caplog.text + + @coroutine_test + async def test_crawl_multiple( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: settings_dict = get_reactor_settings() runner_cls = ( CrawlerRunner @@ -413,13 +457,13 @@ with multiples lines runner = runner_cls(settings_dict) runner.crawl( SimpleSpider, - self.mockserver.url("/status?n=200"), - mockserver=self.mockserver, + mockserver.url("/status?n=200"), + mockserver=mockserver, ) runner.crawl( SimpleSpider, - self.mockserver.url("/status?n=503"), - mockserver=self.mockserver, + mockserver.url("/status?n=503"), + mockserver=mockserver, ) with caplog.at_level(logging.DEBUG): @@ -431,25 +475,15 @@ with multiples lines @coroutine_test async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None: crawler = get_crawler(SimpleSpider) - await maybe_deferred_to_future(crawler.crawl("foo://bar")) + await crawler.crawl_async("foo://bar") assert "NotSupported: Unsupported URL scheme 'foo'" in caplog.text class TestCrawlSpider: - mockserver: MockServer - - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - + @staticmethod async def _run_spider( - self, spider_cls: type[Spider] - ) -> tuple[LogCapture, list[Any], StatsCollector]: + spider_cls: type[Spider], mockserver: MockServer + ) -> tuple[list[Any], StatsCollector]: items = [] def _on_item_scraped(item): @@ -457,103 +491,119 @@ class TestCrawlSpider: crawler = get_crawler(spider_cls) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - await maybe_deferred_to_future( - crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver - ) - ) + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver + ) assert crawler.stats - return log, items, crawler.stats + return items, crawler.stats - @inline_callbacks_test - def test_crawlspider_with_parse(self): + @coroutine_test + async def test_crawlspider_with_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithParseMethod) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse] status 200 (foo: None)" in str(log) - assert "[parse] status 201 (foo: None)" in str(log) - assert "[parse] status 202 (foo: bar)" in str(log) + assert "[parse] status 200 (foo: None)" in caplog.text + assert "[parse] status 201 (foo: None)" in caplog.text + assert "[parse] status 202 (foo: bar)" in caplog.text - @inline_callbacks_test - def test_crawlspider_with_async_callback(self): + @coroutine_test + async def test_crawlspider_with_async_callback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithAsyncCallback) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse_async] status 200 (foo: None)" in str(log) - assert "[parse_async] status 201 (foo: None)" in str(log) - assert "[parse_async] status 202 (foo: bar)" in str(log) + assert "[parse_async] status 200 (foo: None)" in caplog.text + assert "[parse_async] status 201 (foo: None)" in caplog.text + assert "[parse_async] status 202 (foo: bar)" in caplog.text - @inline_callbacks_test - def test_crawlspider_with_async_generator_callback(self): + @coroutine_test + async def test_crawlspider_with_async_generator_callback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse_async_gen] status 200 (foo: None)" in str(log) - assert "[parse_async_gen] status 201 (foo: None)" in str(log) - assert "[parse_async_gen] status 202 (foo: bar)" in str(log) + assert "[parse_async_gen] status 200 (foo: None)" in caplog.text + assert "[parse_async_gen] status 201 (foo: None)" in caplog.text + assert "[parse_async_gen] status 202 (foo: bar)" in caplog.text - @inline_callbacks_test - def test_crawlspider_with_errback(self): + @coroutine_test + async def test_crawlspider_with_errback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithErrback) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse] status 200 (foo: None)" in str(log) - assert "[parse] status 201 (foo: None)" in str(log) - assert "[parse] status 202 (foo: bar)" in str(log) - assert "[errback] status 404" in str(log) - assert "[errback] status 500" in str(log) - assert "[errback] status 501" in str(log) + assert "[parse] status 200 (foo: None)" in caplog.text + assert "[parse] status 201 (foo: None)" in caplog.text + assert "[parse] status 202 (foo: bar)" in caplog.text + assert "[errback] status 404" in caplog.text + assert "[errback] status 500" in caplog.text + assert "[errback] status 501" in caplog.text - @inline_callbacks_test - def test_crawlspider_process_request_cb_kwargs(self): + @coroutine_test + async def test_crawlspider_process_request_cb_kwargs( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) - assert "[parse] status 200 (foo: process_request)" in str(log) - assert "[parse] status 201 (foo: process_request)" in str(log) - assert "[parse] status 202 (foo: bar)" in str(log) + assert "[parse] status 200 (foo: process_request)" in caplog.text + assert "[parse] status 201 (foo: process_request)" in caplog.text + assert "[parse] status 202 (foo: bar)" in caplog.text - @inline_callbacks_test - def test_async_def_parse(self): + @coroutine_test + async def test_async_def_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(AsyncDefSpider) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) - assert "Got response 200" in str(log) + assert "Got response 200" in caplog.text @pytest.mark.only_asyncio - @inline_callbacks_test - def test_async_def_asyncio_parse(self): + @coroutine_test + async def test_async_def_asyncio_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler( AsyncDefAsyncioSpider, { "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor" }, ) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) - assert "Got response 200" in str(log) + assert "Got response 200" in caplog.text @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncio_parse_items_list(self): - log, items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider) - assert "Got response 200" in str(log) + async def test_async_def_asyncio_parse_items_list( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider, mockserver) + assert "Got response 200" in caplog.text assert {"id": 1} in items assert {"id": 2} in items @pytest.mark.only_asyncio - @inline_callbacks_test - def test_async_def_asyncio_parse_items_single_element(self): + @coroutine_test + async def test_async_def_asyncio_parse_items_single_element( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: items = [] def _on_item_scraped(item): @@ -561,26 +611,34 @@ class TestCrawlSpider: crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) - with LogCapture() as log: - yield crawler.crawl( - self.mockserver.url("/status?n=200"), mockserver=self.mockserver + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + mockserver.url("/status?n=200"), mockserver=mockserver ) - assert "Got response 200" in str(log) + assert "Got response 200" in caplog.text assert {"foo": 42} in items @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncgen_parse(self): - log, _, stats = await self._run_spider(AsyncDefAsyncioGenSpider) - assert "Got response 200" in str(log) + async def test_async_def_asyncgen_parse( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + _, stats = await self._run_spider(AsyncDefAsyncioGenSpider, mockserver) + assert "Got response 200" in caplog.text itemcount = stats.get_value("item_scraped_count") assert itemcount == 1 @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncgen_parse_loop(self): - log, items, stats = await self._run_spider(AsyncDefAsyncioGenLoopSpider) - assert "Got response 200" in str(log) + async def test_async_def_asyncgen_parse_loop( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + items, stats = await self._run_spider( + AsyncDefAsyncioGenLoopSpider, mockserver + ) + assert "Got response 200" in caplog.text itemcount = stats.get_value("item_scraped_count") assert itemcount == 10 for i in range(10): @@ -588,11 +646,15 @@ class TestCrawlSpider: @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncgen_parse_exc(self): - log, items, stats = await self._run_spider(AsyncDefAsyncioGenExcSpider) - log = str(log) - assert "Spider error processing" in log - assert "ValueError" in log + async def test_async_def_asyncgen_parse_exc( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + items, stats = await self._run_spider( + AsyncDefAsyncioGenExcSpider, mockserver + ) + assert "Spider error processing" in caplog.text + assert "ValueError" in caplog.text itemcount = stats.get_value("item_scraped_count") assert itemcount == 7 for i in range(7): @@ -600,8 +662,12 @@ class TestCrawlSpider: @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncgen_parse_complex(self): - _, items, stats = await self._run_spider(AsyncDefAsyncioGenComplexSpider) + async def test_async_def_asyncgen_parse_complex( + self, mockserver: MockServer + ) -> None: + items, stats = await self._run_spider( + AsyncDefAsyncioGenComplexSpider, mockserver + ) itemcount = stats.get_value("item_scraped_count") assert itemcount == 156 # some random items @@ -612,40 +678,43 @@ class TestCrawlSpider: @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_asyncio_parse_reqs_list(self): - log, *_ = await self._run_spider(AsyncDefAsyncioReqsReturnSpider) + async def test_async_def_asyncio_parse_reqs_list( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: + with caplog.at_level(logging.INFO): + await self._run_spider(AsyncDefAsyncioReqsReturnSpider, mockserver) for req_id in range(3): - assert f"Got response 200, req_id {req_id}" in str(log) + assert f"Got response 200, req_id {req_id}" in caplog.text @pytest.mark.only_not_asyncio @coroutine_test - async def test_async_def_deferred_direct(self): - _, items, _ = await self._run_spider(AsyncDefDeferredDirectSpider) + async def test_async_def_deferred_direct(self, mockserver: MockServer) -> None: + items, _ = await self._run_spider(AsyncDefDeferredDirectSpider, mockserver) assert items == [{"code": 200}] @pytest.mark.only_asyncio @coroutine_test - async def test_async_def_deferred_wrapped(self): - _, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider) + async def test_async_def_deferred_wrapped(self, mockserver: MockServer) -> None: + items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider, mockserver) assert items == [{"code": 200}] @coroutine_test - async def test_async_def_deferred_maybe_wrapped(self): - _, items, _ = await self._run_spider(AsyncDefDeferredMaybeWrappedSpider) + async def test_async_def_deferred_maybe_wrapped( + self, mockserver: MockServer + ) -> None: + items, _ = await self._run_spider( + AsyncDefDeferredMaybeWrappedSpider, mockserver + ) assert items == [{"code": 200}] - @inline_callbacks_test - def test_response_ssl_certificate_none(self): + @coroutine_test + async def test_response_ssl_certificate_none(self, mockserver: MockServer) -> None: crawler = get_crawler(SingleRequestSpider) - url = self.mockserver.url("/echo?body=test", is_secure=False) - yield crawler.crawl(seed=url, mockserver=self.mockserver) + url = mockserver.url("/echo?body=test", is_secure=False) + await crawler.crawl_async(seed=url, mockserver=mockserver) + assert isinstance(crawler.spider, SingleRequestSpider) assert crawler.spider.meta["responses"][0].certificate is None - @pytest.mark.xfail( - 'config.getoption("--reactor") == "none"', - reason="Not implemented in HttpxDownloadHandler", - strict=True, - ) @pytest.mark.parametrize( "url", [ @@ -669,9 +738,14 @@ class TestCrawlSpider: await crawler.crawl_async(seed=url, mockserver=mockserver) assert isinstance(crawler.spider, SingleRequestSpider) cert = crawler.spider.meta["responses"][0].certificate - assert isinstance(cert, Certificate) - assert cert.getSubject().commonName == b"localhost" - assert cert.getIssuer().commonName == b"localhost" + assert cert is not None + if isinstance(cert, Certificate): # Twisted + assert cert.getSubject().commonName == b"localhost" + assert cert.getIssuer().commonName == b"localhost" + elif isinstance(cert, bytes): # DER bytes + cert_x509 = load_der_x509_certificate(cert) + assert cert_x509.subject.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE" + assert cert_x509.issuer.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE" @pytest.mark.parametrize( "url", @@ -698,10 +772,13 @@ class TestCrawlSpider: assert isinstance(ip_address, IPv4Address) assert str(ip_address) == gethostbyname(expected_netloc) - @inline_callbacks_test - def test_bytes_received_stop_download_callback(self): + @coroutine_test + async def test_bytes_received_stop_download_callback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(BytesReceivedCallbackSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, BytesReceivedCallbackSpider) assert crawler.spider.meta.get("failure") is None assert isinstance(crawler.spider.meta["response"], Response) assert crawler.spider.meta["response"].body == crawler.spider.meta.get( @@ -712,10 +789,13 @@ class TestCrawlSpider: < crawler.spider.full_response_length ) - @inline_callbacks_test - def test_bytes_received_stop_download_errback(self): + @coroutine_test + async def test_bytes_received_stop_download_errback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(BytesReceivedErrbackSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, BytesReceivedErrbackSpider) assert crawler.spider.meta.get("response") is None assert isinstance(crawler.spider.meta["failure"], Failure) assert isinstance(crawler.spider.meta["failure"].value, StopDownload) @@ -728,20 +808,26 @@ class TestCrawlSpider: < crawler.spider.full_response_length ) - @inline_callbacks_test - def test_headers_received_stop_download_callback(self): + @coroutine_test + async def test_headers_received_stop_download_callback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(HeadersReceivedCallbackSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, HeadersReceivedCallbackSpider) assert crawler.spider.meta.get("failure") is None assert isinstance(crawler.spider.meta["response"], Response) assert crawler.spider.meta["response"].headers == crawler.spider.meta.get( "headers_received" ) - @inline_callbacks_test - def test_headers_received_stop_download_errback(self): + @coroutine_test + async def test_headers_received_stop_download_errback( + self, mockserver: MockServer + ) -> None: crawler = get_crawler(HeadersReceivedErrbackSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, HeadersReceivedErrbackSpider) assert crawler.spider.meta.get("response") is None assert isinstance(crawler.spider.meta["failure"], Failure) assert isinstance(crawler.spider.meta["failure"].value, StopDownload) @@ -750,8 +836,10 @@ class TestCrawlSpider: "failure" ].value.response.headers == crawler.spider.meta.get("headers_received") - @inline_callbacks_test - def test_spider_callback_deferred_deprecated(self): + @coroutine_test + async def test_spider_callback_deferred_deprecated( + self, mockserver: MockServer + ) -> None: def cb(response: Response) -> Any: return succeed(None) @@ -760,10 +848,12 @@ class TestCrawlSpider: ScrapyDeprecationWarning, match="Returning Deferreds from spider callbacks is deprecated", ): - yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) + await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb) - @inline_callbacks_test - def test_spider_errback(self): + @coroutine_test + async def test_spider_errback( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: failures = [] def eb(failure: Failure) -> Failure: @@ -771,72 +861,82 @@ class TestCrawlSpider: return failure crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) assert len(failures) == 1 - assert "HTTP status code is not handled or not allowed" in str(log) - assert "Spider error processing" not in str(log) + assert "HTTP status code is not handled or not allowed" in caplog.text + assert "Spider error processing" not in caplog.text - @inline_callbacks_test - def test_spider_errback_silence(self): + @coroutine_test + async def test_spider_errback_silence( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: failures = [] def eb(failure: Failure) -> None: failures.append(failure) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) assert len(failures) == 1 - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text - @inline_callbacks_test - def test_spider_errback_exception(self): + @coroutine_test + async def test_spider_errback_exception( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> None: raise ValueError("foo") crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) - assert "Spider error processing" in str(log) + assert "Spider error processing" in caplog.text - @inline_callbacks_test - def test_spider_errback_item(self): + @coroutine_test + async def test_spider_errback_item( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Any: return {"foo": "bar"} crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) - assert "'item_scraped_count': 1" in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "'item_scraped_count': 1" in caplog.text - @inline_callbacks_test - def test_spider_errback_request(self): + @coroutine_test + async def test_spider_errback_request( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Request: - return Request(self.mockserver.url("/")) + return Request(mockserver.url("/")) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) - assert "Crawled (200)" in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "Crawled (200)" in caplog.text - @inline_callbacks_test - def test_spider_errback_downloader_error(self): + @coroutine_test + async def test_spider_errback_downloader_error( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: failures = [] def eb(failure: Failure) -> Failure: @@ -844,57 +944,65 @@ class TestCrawlSpider: return failure crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb ) assert len(failures) == 1 - assert "Error downloading" in str(log) - assert "Spider error processing" not in str(log) + assert "Error downloading" in caplog.text + assert "Spider error processing" not in caplog.text - @inline_callbacks_test - def test_spider_errback_downloader_error_exception(self): + @coroutine_test + async def test_spider_errback_downloader_error_exception( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> None: raise ValueError("foo") crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb ) - assert "Error downloading" in str(log) - assert "Spider error processing" in str(log) + assert "Error downloading" in caplog.text + assert "Spider error processing" in caplog.text - @inline_callbacks_test - def test_spider_errback_downloader_error_item(self): + @coroutine_test + async def test_spider_errback_downloader_error_item( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Any: return {"foo": "bar"} crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + with caplog.at_level(logging.INFO): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb ) - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) - assert "'item_scraped_count': 1" in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "'item_scraped_count': 1" in caplog.text - @inline_callbacks_test - def test_spider_errback_downloader_error_request(self): + @coroutine_test + async def test_spider_errback_downloader_error_request( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Request: - return Request(self.mockserver.url("/")) + return Request(mockserver.url("/")) crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl( - seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/drop?abort=1"), errback_func=eb ) - assert "HTTP status code is not handled or not allowed" not in str(log) - assert "Spider error processing" not in str(log) - assert "Crawled (200)" in str(log) + assert "HTTP status code is not handled or not allowed" not in caplog.text + assert "Spider error processing" not in caplog.text + assert "Crawled (200)" in caplog.text - @inline_callbacks_test - def test_spider_errback_deferred_deprecated(self): + @coroutine_test + async def test_spider_errback_deferred_deprecated( + self, mockserver: MockServer + ) -> None: def eb(failure: Failure) -> Any: return succeed(None) @@ -903,28 +1011,32 @@ class TestCrawlSpider: ScrapyDeprecationWarning, match="Returning Deferreds from spider errbacks is deprecated", ): - yield crawler.crawl( - seed=self.mockserver.url("/status?n=400"), errback_func=eb + await crawler.crawl_async( + seed=mockserver.url("/status?n=400"), errback_func=eb ) - @inline_callbacks_test - def test_raise_closespider(self): + @coroutine_test + async def test_raise_closespider( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def cb(response): raise CloseSpider crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) - assert "Closing spider (cancelled)" in str(log) - assert "Spider error processing" not in str(log) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb) + assert "Closing spider (cancelled)" in caplog.text + assert "Spider error processing" not in caplog.text - @inline_callbacks_test - def test_raise_closespider_reason(self): + @coroutine_test + async def test_raise_closespider_reason( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: def cb(response): raise CloseSpider("my_reason") crawler = get_crawler(SingleRequestSpider) - with LogCapture() as log: - yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb) - assert "Closing spider (my_reason)" in str(log) - assert "Spider error processing" not in str(log) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb) + assert "Closing spider (my_reason)" in caplog.text + assert "Spider error processing" not in caplog.text diff --git a/tests/test_crawler.py b/tests/test_crawler.py index cbcb7e274..82956735b 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -1,13 +1,13 @@ +from __future__ import annotations + import asyncio import logging import re import warnings -from collections.abc import Generator from pathlib import Path -from typing import Any, cast +from typing import Any, ClassVar import pytest -from twisted.internet.defer import Deferred from zope.interface.exceptions import MultipleInvalid import scrapy @@ -23,7 +23,7 @@ from scrapy.crawler import ( from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings -from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future +from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future from scrapy.utils.log import ( _uninstall_scrapy_root_handler, configure_logging, @@ -31,12 +31,14 @@ from scrapy.utils.log import ( ) from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings -from tests.utils.decorators import coroutine_test, inline_callbacks_test +from tests.utils.decorators import coroutine_test BASE_SETTINGS: dict[str, Any] = {} -def get_raw_crawler(spidercls=None, settings_dict=None): +def get_raw_crawler( + spidercls: type[Spider] | None = None, settings_dict: dict[str, Any] | None = None +) -> Crawler: """get_crawler alternative that only calls the __init__ method of the crawler.""" settings = Settings() @@ -46,14 +48,18 @@ def get_raw_crawler(spidercls=None, settings_dict=None): class TestBaseCrawler: - def assertOptionIsDefault(self, settings: Settings, key: str) -> None: + @staticmethod + def assertOptionIsDefault(settings: Settings, key: str) -> None: assert isinstance(settings, Settings) assert settings[key] == getattr(default_settings, key) class TestCrawler(TestBaseCrawler): - def test_populate_spidercls_settings(self): - spider_settings = {"TEST1": "spider", "TEST2": "spider"} + def test_populate_spidercls_settings(self) -> None: + spider_settings: dict[str, Any] = { + "TEST1": "spider", + "TEST2": "spider", + } project_settings = { **BASE_SETTINGS, "TEST1": "project", @@ -76,47 +82,47 @@ class TestCrawler(TestBaseCrawler): assert not settings.frozen assert crawler.settings.frozen - def test_crawler_accepts_dict(self): + def test_crawler_accepts_dict(self) -> None: crawler = get_crawler(DefaultSpider, {"foo": "bar"}) assert crawler.settings["foo"] == "bar" self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") - def test_crawler_accepts_None(self): + def test_crawler_accepts_None(self) -> None: with warnings.catch_warnings(): warnings.simplefilter("ignore", ScrapyDeprecationWarning) crawler = Crawler(DefaultSpider) self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED") - def test_crawler_rejects_spider_objects(self): + def test_crawler_rejects_spider_objects(self) -> None: with pytest.raises(ValueError, match="spidercls argument must be a class"): - Crawler(DefaultSpider()) - - @inline_callbacks_test - def test_crawler_crawl_twice_seq_unsupported(self): - crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) - yield crawler.crawl() - with pytest.raises(RuntimeError, match="more than once on the same instance"): - yield crawler.crawl() + Crawler(DefaultSpider()) # type: ignore[arg-type] @coroutine_test - async def test_crawler_crawl_async_twice_seq_unsupported(self): + async def test_crawler_crawl_twice_seq_unsupported(self) -> None: + crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) + await maybe_deferred_to_future(crawler.crawl()) + with pytest.raises(RuntimeError, match="more than once on the same instance"): + await maybe_deferred_to_future(crawler.crawl()) + + @coroutine_test + async def test_crawler_crawl_async_twice_seq_unsupported(self) -> None: crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) await crawler.crawl_async() with pytest.raises(RuntimeError, match="more than once on the same instance"): await crawler.crawl_async() - @inline_callbacks_test - def test_crawler_crawl_twice_parallel_unsupported(self): + @coroutine_test + async def test_crawler_crawl_twice_parallel_unsupported(self) -> None: crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) d1 = crawler.crawl() d2 = crawler.crawl() - yield d1 + await maybe_deferred_to_future(d1) with pytest.raises(RuntimeError, match="Crawling already taking place"): - yield d2 + await maybe_deferred_to_future(d2) @pytest.mark.only_asyncio @coroutine_test - async def test_crawler_crawl_async_twice_parallel_unsupported(self): + async def test_crawler_crawl_async_twice_parallel_unsupported(self) -> None: crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS) t1 = asyncio.create_task(crawler.crawl_async()) t2 = asyncio.create_task(crawler.crawl_async()) @@ -124,12 +130,12 @@ class TestCrawler(TestBaseCrawler): with pytest.raises(RuntimeError, match="Crawling already taking place"): await t2 - def test_get_addon(self): + def test_get_addon(self) -> None: class ParentAddon: pass class TrackingAddon(ParentAddon): - instances = [] + instances: ClassVar[list[TrackingAddon]] = [] def __init__(self): TrackingAddon.instances.append(self) @@ -150,7 +156,7 @@ class TestCrawler(TestBaseCrawler): addon = crawler.get_addon(TrackingAddon) assert addon == expected - addon = crawler.get_addon(DefaultSpider) + addon = crawler.get_addon(DefaultSpider) # type: ignore[assignment] assert addon is None addon = crawler.get_addon(ParentAddon) @@ -162,19 +168,21 @@ class TestCrawler(TestBaseCrawler): addon = crawler.get_addon(ChildAddon) assert addon is None - @inline_callbacks_test - def test_get_downloader_middleware(self): + @coroutine_test + async def test_get_downloader_middleware(self) -> None: class ParentDownloaderMiddleware: pass class TrackingDownloaderMiddleware(ParentDownloaderMiddleware): - instances = [] + instances: ClassVar[list[TrackingDownloaderMiddleware]] = [] def __init__(self): TrackingDownloaderMiddleware.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): @@ -198,18 +206,18 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingDownloaderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert len(TrackingDownloaderMiddleware.instances) == 1 assert MySpider.result == TrackingDownloaderMiddleware.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentDownloaderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result == TrackingDownloaderMiddleware.instances[-1] class ChildDownloaderMiddleware(TrackingDownloaderMiddleware): @@ -217,16 +225,16 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildDownloaderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None - def test_get_downloader_middleware_not_crawling(self): + def test_get_downloader_middleware_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) with pytest.raises(RuntimeError): crawler.get_downloader_middleware(DefaultSpider) - @inline_callbacks_test - def test_get_downloader_middleware_no_engine(self): + @coroutine_test + async def test_get_downloader_middleware_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @@ -240,21 +248,23 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): - yield crawler.crawl() + await crawler.crawl_async() - @inline_callbacks_test - def test_get_extension(self): + @coroutine_test + async def test_get_extension(self) -> None: class ParentExtension: pass class TrackingExtension(ParentExtension): - instances = [] + instances: ClassVar[list[TrackingExtension]] = [] def __init__(self): TrackingExtension.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): @@ -278,18 +288,18 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingExtension - yield crawler.crawl() + await crawler.crawl_async() assert len(TrackingExtension.instances) == 1 assert MySpider.result == TrackingExtension.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentExtension - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result == TrackingExtension.instances[-1] class ChildExtension(TrackingExtension): @@ -297,16 +307,16 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildExtension - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None - def test_get_extension_not_crawling(self): + def test_get_extension_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) with pytest.raises(RuntimeError): crawler.get_extension(DefaultSpider) - @inline_callbacks_test - def test_get_extension_no_engine(self): + @coroutine_test + async def test_get_extension_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @@ -320,21 +330,23 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): - yield crawler.crawl() + await crawler.crawl_async() - @inline_callbacks_test - def test_get_item_pipeline(self): + @coroutine_test + async def test_get_item_pipeline(self) -> None: class ParentItemPipeline: pass class TrackingItemPipeline(ParentItemPipeline): - instances = [] + instances: ClassVar[list[TrackingItemPipeline]] = [] def __init__(self): TrackingItemPipeline.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): @@ -358,18 +370,18 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingItemPipeline - yield crawler.crawl() + await crawler.crawl_async() assert len(TrackingItemPipeline.instances) == 1 assert MySpider.result == TrackingItemPipeline.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentItemPipeline - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result == TrackingItemPipeline.instances[-1] class ChildItemPipeline(TrackingItemPipeline): @@ -377,16 +389,16 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildItemPipeline - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None - def test_get_item_pipeline_not_crawling(self): + def test_get_item_pipeline_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) with pytest.raises(RuntimeError): crawler.get_item_pipeline(DefaultSpider) - @inline_callbacks_test - def test_get_item_pipeline_no_engine(self): + @coroutine_test + async def test_get_item_pipeline_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @@ -400,21 +412,23 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): - yield crawler.crawl() + await crawler.crawl_async() - @inline_callbacks_test - def test_get_spider_middleware(self): + @coroutine_test + async def test_get_spider_middleware(self) -> None: class ParentSpiderMiddleware: pass class TrackingSpiderMiddleware(ParentSpiderMiddleware): - instances = [] + instances: ClassVar[list[TrackingSpiderMiddleware]] = [] def __init__(self): TrackingSpiderMiddleware.instances.append(self) class MySpider(Spider): name = "myspider" + cls: ClassVar[type[Any]] + result: ClassVar[Any] @classmethod def from_crawler(cls, crawler): @@ -438,18 +452,18 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = TrackingSpiderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert len(TrackingSpiderMiddleware.instances) == 1 assert MySpider.result == TrackingSpiderMiddleware.instances[-1] crawler = get_raw_crawler(MySpider, settings) MySpider.cls = DefaultSpider - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ParentSpiderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result == TrackingSpiderMiddleware.instances[-1] class ChildSpiderMiddleware(TrackingSpiderMiddleware): @@ -457,16 +471,16 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, settings) MySpider.cls = ChildSpiderMiddleware - yield crawler.crawl() + await crawler.crawl_async() assert MySpider.result is None - def test_get_spider_middleware_not_crawling(self): + def test_get_spider_middleware_not_crawling(self) -> None: crawler = get_raw_crawler(settings_dict=BASE_SETTINGS) with pytest.raises(RuntimeError): crawler.get_spider_middleware(DefaultSpider) - @inline_callbacks_test - def test_get_spider_middleware_no_engine(self): + @coroutine_test + async def test_get_spider_middleware_no_engine(self) -> None: class MySpider(Spider): name = "myspider" @@ -480,22 +494,23 @@ class TestCrawler(TestBaseCrawler): crawler = get_raw_crawler(MySpider, BASE_SETTINGS) with pytest.raises(RuntimeError): - yield crawler.crawl() + await crawler.crawl_async() class TestSpiderSettings: - def test_spider_custom_settings(self): + def test_spider_custom_settings(self) -> None: class MySpider(scrapy.Spider): name = "spider" custom_settings = {"AUTOTHROTTLE_ENABLED": True} crawler = get_crawler(MySpider) + assert crawler.extensions enabled_exts = [e.__class__ for e in crawler.extensions.middlewares] assert AutoThrottle in enabled_exts class TestCrawlerLogging: - def test_no_root_handler_installed(self): + def test_no_root_handler_installed(self) -> None: handler = get_scrapy_root_handler() if handler is not None: logging.root.removeHandler(handler) @@ -507,7 +522,7 @@ class TestCrawlerLogging: assert get_scrapy_root_handler() is None @coroutine_test - async def test_spider_custom_settings_log_level(self, tmp_path): + async def test_spider_custom_settings_log_level(self, tmp_path: Path) -> None: log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") @@ -535,9 +550,13 @@ class TestCrawlerLogging: try: configure_logging() - assert get_scrapy_root_handler().level == logging.DEBUG + handler = get_scrapy_root_handler() + assert handler is not None + assert handler.level == logging.DEBUG crawler = get_crawler(MySpider) - assert get_scrapy_root_handler().level == logging.INFO + handler = get_scrapy_root_handler() + assert handler is not None + assert handler.level == logging.INFO await crawler.crawl_async() finally: _uninstall_scrapy_root_handler() @@ -549,12 +568,13 @@ class TestCrawlerLogging: assert "info message" in logged assert "warning message" in logged assert "error message" in logged + assert crawler.stats assert crawler.stats.get_value("log_count/ERROR") == 1 assert crawler.stats.get_value("log_count/WARNING") == 1 assert info_count == 1 assert crawler.stats.get_value("log_count/DEBUG", 0) == 0 - def test_spider_custom_settings_log_append(self, tmp_path): + def test_spider_custom_settings_log_append(self, tmp_path: Path) -> None: log_file = Path(tmp_path, "log.txt") log_file.write_text("previous message\n", encoding="utf-8") @@ -579,12 +599,12 @@ class TestCrawlerLogging: class SpiderLoaderWithWrongInterface: - def unneeded_method(self): + def unneeded_method(self) -> None: pass class TestCrawlerRunner(TestBaseCrawler): - def test_spider_manager_verify_interface(self): + def test_spider_manager_verify_interface(self) -> None: settings = Settings( { "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, @@ -593,18 +613,18 @@ class TestCrawlerRunner(TestBaseCrawler): with pytest.raises(MultipleInvalid): CrawlerRunner(settings) - def test_crawler_runner_accepts_dict(self): + def test_crawler_runner_accepts_dict(self) -> None: runner = CrawlerRunner({"foo": "bar"}) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_crawler_runner_accepts_None(self): + def test_crawler_runner_accepts_None(self) -> None: runner = CrawlerRunner() self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") class TestAsyncCrawlerRunner(TestBaseCrawler): - def test_spider_manager_verify_interface(self): + def test_spider_manager_verify_interface(self) -> None: settings = Settings( { "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface, @@ -613,23 +633,23 @@ class TestAsyncCrawlerRunner(TestBaseCrawler): with pytest.raises(MultipleInvalid): AsyncCrawlerRunner(settings) - def test_crawler_runner_accepts_dict(self): + def test_crawler_runner_accepts_dict(self) -> None: runner = AsyncCrawlerRunner({"foo": "bar"}) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_crawler_runner_accepts_None(self): + def test_crawler_runner_accepts_None(self) -> None: runner = AsyncCrawlerRunner() self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") class TestCrawlerProcess(TestBaseCrawler): - def test_crawler_process_accepts_dict(self): + def test_crawler_process_accepts_dict(self) -> None: runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_crawler_process_accepts_None(self): + def test_crawler_process_accepts_None(self) -> None: runner = CrawlerProcess(install_root_handler=False) self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") @@ -668,33 +688,35 @@ class NoRequestsSpider(scrapy.Spider): @pytest.mark.requires_reactor # CrawlerRunner requires a reactor class TestCrawlerRunnerHasSpider: - @staticmethod - def _runner() -> CrawlerRunnerBase: + @pytest.fixture + def runner(self) -> CrawlerRunnerBase: return CrawlerRunner(get_reactor_settings()) @staticmethod - def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> Deferred[None]: - return cast("Deferred[None]", runner.crawl(spider)) + async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None: + await ensure_awaitable(runner.crawl(spider)) - @inline_callbacks_test - def test_crawler_runner_bootstrap_successful(self): - runner = self._runner() - yield self._crawl(runner, NoRequestsSpider) + @coroutine_test + async def test_crawler_runner_bootstrap_successful( + self, runner: CrawlerRunnerBase + ) -> None: + await self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed - @inline_callbacks_test - def test_crawler_runner_bootstrap_successful_for_several(self): - runner = self._runner() - yield self._crawl(runner, NoRequestsSpider) - yield self._crawl(runner, NoRequestsSpider) + @coroutine_test + async def test_crawler_runner_bootstrap_successful_for_several( + self, runner: CrawlerRunnerBase + ) -> None: + await self._crawl(runner, NoRequestsSpider) + await self._crawl(runner, NoRequestsSpider) assert not runner.bootstrap_failed - @inline_callbacks_test - def test_crawler_runner_bootstrap_failed(self): - runner = self._runner() - + @coroutine_test + async def test_crawler_runner_bootstrap_failed( + self, runner: CrawlerRunnerBase + ) -> None: try: - yield self._crawl(runner, ExceptionSpider) + await self._crawl(runner, ExceptionSpider) except ValueError: pass else: @@ -702,25 +724,25 @@ class TestCrawlerRunnerHasSpider: assert runner.bootstrap_failed - @inline_callbacks_test - def test_crawler_runner_bootstrap_failed_for_several(self): - runner = self._runner() - + @coroutine_test + async def test_crawler_runner_bootstrap_failed_for_several( + self, runner: CrawlerRunnerBase + ) -> None: try: - yield self._crawl(runner, ExceptionSpider) + await self._crawl(runner, ExceptionSpider) except ValueError: pass else: pytest.fail("Exception should be raised from spider") - yield self._crawl(runner, NoRequestsSpider) + await self._crawl(runner, NoRequestsSpider) assert runner.bootstrap_failed - @inline_callbacks_test - def test_crawler_runner_asyncio_enabled_true( + @coroutine_test + async def test_crawler_runner_asyncio_enabled_true( self, reactor_pytest: str - ) -> Generator[Deferred[Any], Any, None]: + ) -> None: if reactor_pytest != "asyncio": runner = CrawlerRunner( settings={ @@ -731,7 +753,7 @@ class TestCrawlerRunnerHasSpider: Exception, match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)", ): - yield self._crawl(runner, NoRequestsSpider) + await self._crawl(runner, NoRequestsSpider) else: CrawlerRunner( settings={ @@ -742,15 +764,11 @@ class TestCrawlerRunnerHasSpider: @pytest.mark.only_asyncio class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): - @staticmethod - def _runner() -> CrawlerRunnerBase: + @pytest.fixture + def runner(self) -> CrawlerRunnerBase: return AsyncCrawlerRunner(get_reactor_settings()) - @staticmethod - def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> Deferred[None]: - return deferred_from_coro(runner.crawl(spider)) - - def test_crawler_runner_asyncio_enabled_true(self): + def test_crawler_runner_asyncio_enabled_true(self) -> None: # type: ignore[override] pytest.skip("This test is only for CrawlerRunner") @@ -762,7 +780,9 @@ class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider): ({"LOG_VERSIONS": []}, None), ], ) -def test_log_scrapy_info(settings, items, caplog): +def test_log_scrapy_info( + settings: dict[str, Any], items: list[str] | None, caplog: pytest.LogCaptureFixture +) -> None: with caplog.at_level("INFO"): CrawlerProcess(settings, install_root_handler=False) assert ( diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index beae4f277..146d94ecd 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -52,7 +52,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): with the same file names and expectations. """ - def test_simple(self): + def test_simple(self) -> None: log = self.run_script("simple.py") assert "Spider closed (finished)" in log assert ( @@ -61,7 +61,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): ) assert "is_reactorless(): False" in log - def test_multi(self): + def test_multi(self) -> None: log = self.run_script("multi.py") assert "Spider closed (finished)" in log assert ( @@ -70,7 +70,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): ) assert "ReactorAlreadyInstalledError" not in log - def test_reactor_default(self): + def test_reactor_default(self) -> None: log = self.run_script("reactor_default.py") assert "Spider closed (finished)" not in log assert ( @@ -78,7 +78,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" ) in log - def test_asyncio_enabled_no_reactor(self): + def test_asyncio_enabled_no_reactor(self) -> None: log = self.run_script("asyncio_enabled_no_reactor.py") assert "Spider closed (finished)" in log assert ( @@ -87,7 +87,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): ) assert "RuntimeError" not in log - def test_asyncio_enabled_reactor(self): + def test_asyncio_enabled_reactor(self) -> None: log = self.run_script("asyncio_enabled_reactor.py") assert "Spider closed (finished)" in log assert ( @@ -100,7 +100,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): parse_version(w3lib_version) >= parse_version("2.0.0"), reason="w3lib 2.0.0 and later do not allow invalid domains.", ) - def test_ipv6_default_name_resolver(self): + def test_ipv6_default_name_resolver(self) -> None: log = self.run_script("default_name_resolver.py") assert "Spider closed (finished)" in log assert ( @@ -112,7 +112,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): in log ) - def test_caching_hostname_resolver_ipv6(self): + def test_caching_hostname_resolver_ipv6(self) -> None: log = self.run_script("caching_hostname_resolver_ipv6.py") assert "Spider closed (finished)" in log assert "scrapy.exceptions.CannotResolveHostError" not in log @@ -126,7 +126,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "TimeoutError" not in log assert "scrapy.exceptions.CannotResolveHostError" not in log - def test_twisted_reactor_asyncio(self): + def test_twisted_reactor_asyncio(self) -> None: log = self.run_script("twisted_reactor_asyncio.py") assert "Spider closed (finished)" in log assert ( @@ -134,7 +134,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): in log ) - def test_twisted_reactor_asyncio_custom_settings(self): + def test_twisted_reactor_asyncio_custom_settings(self) -> None: log = self.run_script("twisted_reactor_custom_settings.py") assert "Spider closed (finished)" in log assert ( @@ -142,7 +142,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): in log ) - def test_twisted_reactor_asyncio_custom_settings_same(self): + def test_twisted_reactor_asyncio_custom_settings_same(self) -> None: log = self.run_script("twisted_reactor_custom_settings_same.py") assert "Spider closed (finished)" in log assert ( @@ -151,7 +151,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): ) @pytest.mark.requires_uvloop - def test_custom_loop_asyncio(self): + def test_custom_loop_asyncio(self) -> None: log = self.run_script("asyncio_custom_loop.py") assert "Spider closed (finished)" in log assert ( @@ -161,7 +161,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "Using asyncio event loop: uvloop.Loop" in log @pytest.mark.requires_uvloop - def test_custom_loop_asyncio_deferred_signal(self): + def test_custom_loop_asyncio_deferred_signal(self) -> None: log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop") assert "Spider closed (finished)" in log assert ( @@ -172,7 +172,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "async pipeline opened!" in log @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_same_loop(self): + def test_asyncio_enabled_reactor_same_loop(self) -> None: log = self.run_script("asyncio_enabled_reactor_same_loop.py") assert "Spider closed (finished)" in log assert ( @@ -182,7 +182,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "Using asyncio event loop: uvloop.Loop" in log @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_different_loop(self): + def test_asyncio_enabled_reactor_different_loop(self) -> None: log = self.run_script("asyncio_enabled_reactor_different_loop.py") assert "Spider closed (finished)" not in log assert ( @@ -190,7 +190,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): "setting (uvloop.Loop)" ) in log - def test_default_loop_asyncio_deferred_signal(self): + def test_default_loop_asyncio_deferred_signal(self) -> None: log = self.run_script("asyncio_deferred_signal.py") assert "Spider closed (finished)" in log assert ( @@ -200,7 +200,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "Using asyncio event loop: uvloop.Loop" not in log assert "async pipeline opened!" in log - def test_args_change_settings(self): + def test_args_change_settings(self) -> None: log = self.run_script("args_settings.py") assert "Spider closed (finished)" in log assert "The value of FOO is 42" in log @@ -243,7 +243,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): def script_dir(self) -> Path: return self.get_script_dir("CrawlerProcess") - def test_reactor_default_twisted_reactor_select(self): + def test_reactor_default_twisted_reactor_select(self) -> None: log = self.run_script("reactor_default_twisted_reactor_select.py") if platform.system() in ["Windows", "Darwin"]: # The goal of this test function is to test that, when a reactor is @@ -264,7 +264,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): "(twisted.internet.selectreactor.SelectReactor)" ) in log - def test_reactor_select(self): + def test_reactor_select(self) -> None: log = self.run_script("reactor_select.py") assert "Spider closed (finished)" not in log assert ( @@ -272,12 +272,12 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)" ) in log - def test_reactor_select_twisted_reactor_select(self): + def test_reactor_select_twisted_reactor_select(self) -> None: log = self.run_script("reactor_select_twisted_reactor_select.py") assert "Spider closed (finished)" in log assert "ReactorAlreadyInstalledError" not in log - def test_reactor_select_subclass_twisted_reactor_select(self): + def test_reactor_select_subclass_twisted_reactor_select(self) -> None: log = self.run_script("reactor_select_subclass_twisted_reactor_select.py") assert "Spider closed (finished)" not in log assert ( @@ -285,7 +285,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): "(twisted.internet.selectreactor.SelectReactor)" ) in log - def test_twisted_reactor_select(self): + def test_twisted_reactor_select(self) -> None: log = self.run_script("twisted_reactor_select.py") assert "Spider closed (finished)" in log assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log @@ -293,12 +293,12 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): @pytest.mark.skipif( platform.system() == "Windows", reason="PollReactor is not supported on Windows" ) - def test_twisted_reactor_poll(self): + def test_twisted_reactor_poll(self) -> None: log = self.run_script("twisted_reactor_poll.py") assert "Spider closed (finished)" in log assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log - def test_twisted_reactor_asyncio_custom_settings_conflict(self): + def test_twisted_reactor_asyncio_custom_settings_conflict(self) -> None: log = self.run_script("twisted_reactor_custom_settings_conflict.py") assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log assert ( @@ -306,7 +306,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): in log ) - def test_reactorless(self): + def test_reactorless(self) -> None: log = self.run_script("reactorless.py") assert ( "RuntimeError: CrawlerProcess doesn't support TWISTED_REACTOR_ENABLED=False" @@ -319,7 +319,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): def script_dir(self) -> Path: return self.get_script_dir("AsyncCrawlerProcess") - def test_twisted_reactor_custom_settings_select(self): + def test_twisted_reactor_custom_settings_select(self) -> None: log = self.run_script("twisted_reactor_custom_settings_select.py") assert "Spider closed (finished)" not in log assert ( @@ -329,7 +329,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): ) in log @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_same_loop(self): + def test_asyncio_enabled_reactor_same_loop(self) -> None: log = self.run_script("asyncio_custom_loop_custom_settings_same.py") assert "Spider closed (finished)" in log assert ( @@ -339,7 +339,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert "Using asyncio event loop: uvloop.Loop" in log @pytest.mark.requires_uvloop - def test_asyncio_enabled_reactor_different_loop(self): + def test_asyncio_enabled_reactor_different_loop(self) -> None: log = self.run_script("asyncio_custom_loop_custom_settings_different.py") assert "Spider closed (finished)" not in log assert ( @@ -347,7 +347,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): "setting (uvloop.Loop)" ) in log - def test_reactorless_simple(self): + def test_reactorless_simple(self) -> None: log = self.run_script("reactorless_simple.py") assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log @@ -356,7 +356,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 assert log.count("WARNING: ") == 2 - def test_reactorless_custom_settings(self): + def test_reactorless_custom_settings(self) -> None: """Setting TWISTED_REACTOR_ENABLED=False in spider settings is not currently supported, AsyncCrawlerProcess will install a reactor in this case. @@ -368,7 +368,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): in log ) - def test_reactorless_datauri(self): + def test_reactorless_datauri(self) -> None: log = self.run_script("reactorless_datauri.py") assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log @@ -378,13 +378,13 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 assert log.count("WARNING: ") == 2 - def test_reactorless_import_hook(self): + def test_reactorless_import_hook(self) -> None: log = self.run_script("reactorless_import_hook.py") assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log assert "ImportError: Import of twisted.internet.reactor is forbidden" in log - def test_reactorless_telnetconsole_default(self): + def test_reactorless_telnetconsole_default(self) -> None: """By default TWISTED_REACTOR_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False.""" log = self.run_script("reactorless_simple.py") # no need for a separate script assert "Not using a Twisted reactor" in log @@ -392,7 +392,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert "The TelnetConsole extension requires a Twisted reactor" not in log assert "scrapy.extensions.telnet.TelnetConsole" not in log - def test_reactorless_telnetconsole_disabled(self): + def test_reactorless_telnetconsole_disabled(self) -> None: """Explicit TELNETCONSOLE_ENABLED=False, there are no warnings.""" log = self.run_script("reactorless_telnetconsole_disabled.py") assert "Not using a Twisted reactor" in log @@ -400,14 +400,14 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase): assert "The TelnetConsole extension requires a Twisted reactor" not in log assert "scrapy.extensions.telnet.TelnetConsole" not in log - def test_reactorless_telnetconsole_enabled(self): + def test_reactorless_telnetconsole_enabled(self) -> None: """Explicit TELNETCONSOLE_ENABLED=True, the user gets a warning.""" log = self.run_script("reactorless_telnetconsole_enabled.py") assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log assert "The TelnetConsole extension requires a Twisted reactor" in log - def test_reactorless_reactor(self): + def test_reactorless_reactor(self) -> None: log = self.run_script("reactorless_reactor.py") assert ( "RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed" @@ -427,7 +427,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): with the same file names and expectations. """ - def test_simple(self): + def test_simple(self) -> None: log = self.run_script("simple.py") assert "Spider closed (finished)" in log assert ( @@ -436,7 +436,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): ) assert "is_reactorless(): False" in log - def test_multi_parallel(self): + def test_multi_parallel(self) -> None: log = self.run_script("multi_parallel.py") assert "Spider closed (finished)" in log assert ( @@ -449,7 +449,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): re.DOTALL, ) - def test_multi_seq(self): + def test_multi_seq(self) -> None: log = self.run_script("multi_seq.py") assert "Spider closed (finished)" in log assert ( @@ -463,7 +463,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): ) @pytest.mark.requires_uvloop - def test_custom_loop_same(self): + def test_custom_loop_same(self) -> None: log = self.run_script("custom_loop_same.py") assert "Spider closed (finished)" in log assert ( @@ -473,7 +473,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): assert "Using asyncio event loop: uvloop.Loop" in log @pytest.mark.requires_uvloop - def test_custom_loop_different(self): + def test_custom_loop_different(self) -> None: log = self.run_script("custom_loop_different.py") assert "Spider closed (finished)" not in log assert ( @@ -481,7 +481,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin): "setting (uvloop.Loop)" ) in log - def test_no_reactor(self): + def test_no_reactor(self) -> None: log = self.run_script("no_reactor.py") assert "Spider closed (finished)" not in log assert ( @@ -495,7 +495,7 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): def script_dir(self) -> Path: return self.get_script_dir("CrawlerRunner") - def test_explicit_default_reactor(self): + def test_explicit_default_reactor(self) -> None: log = self.run_script("explicit_default_reactor.py") assert "Spider closed (finished)" in log assert ( @@ -503,14 +503,14 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): not in log ) - def test_response_ip_address(self): + def test_response_ip_address(self) -> None: log = self.run_script("ip_address.py") assert "INFO: Spider closed (finished)" in log assert "INFO: Host: not.a.real.domain" in log assert "INFO: Type: " in log assert "INFO: IP address: 127.0.0.1" in log - def test_change_default_reactor(self): + def test_change_default_reactor(self) -> None: log = self.run_script("change_reactor.py") assert ( "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" @@ -518,7 +518,7 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): ) assert "DEBUG: Using asyncio event loop" in log - def test_reactorless(self): + def test_reactorless(self) -> None: log = self.run_script("reactorless.py") assert ( "RuntimeError: CrawlerRunner doesn't support TWISTED_REACTOR_ENABLED=False" @@ -531,7 +531,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): def script_dir(self) -> Path: return self.get_script_dir("AsyncCrawlerRunner") - def test_simple_default_reactor(self): + def test_simple_default_reactor(self) -> None: log = self.run_script("simple_default_reactor.py") assert "Spider closed (finished)" not in log assert ( @@ -539,7 +539,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): "AsyncCrawlerRunner requires that the installed Twisted reactor" ) in log - def test_reactorless_simple(self): + def test_reactorless_simple(self) -> None: log = self.run_script("reactorless_simple.py") assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log @@ -548,7 +548,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 assert log.count("WARNING: ") == 2 - def test_reactorless_custom_settings(self): + def test_reactorless_custom_settings(self) -> None: """Setting TWISTED_REACTOR_ENABLED=False in spider settings is not currently supported, AsyncCrawlerRunner will expect a reactor installed by the user. @@ -557,7 +557,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): assert "Spider closed (finished)" not in log assert "We expected a Twisted reactor to be installed but it isn't." in log - def test_reactorless_datauri(self): + def test_reactorless_datauri(self) -> None: log = self.run_script("reactorless_datauri.py") assert "Not using a Twisted reactor" in log assert "Spider closed (finished)" in log @@ -567,7 +567,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase): assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2 assert log.count("WARNING: ") == 2 - def test_reactorless_reactor(self): + def test_reactorless_reactor(self) -> None: log = self.run_script("reactorless_reactor.py") assert ( "RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed" diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index 4436efd9b..578e84359 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -15,14 +15,14 @@ class TestScrapyUtils: See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011 """ - if not os.environ.get("_SCRAPY_PINNED", None): - pytest.skip("Not in a pinned environment") + if not os.environ.get("_SCRAPY_MIN", None): + pytest.skip("Not in a min environment") tox_config_file_path = Path(__file__).parent / ".." / "tox.ini" config_parser = ConfigParser() config_parser.read(tox_config_file_path) pattern = r"Twisted==([\d.]+)" - match = re.search(pattern, config_parser["pinned"]["deps"]) + match = re.search(pattern, config_parser["min"]["deps"]) pinned_twisted_version_string = match[1] assert twisted_version.short() == pinned_twisted_version_string diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py index d4dfe8396..fdacad963 100644 --- a/tests/test_downloader_handler_httpx.py +++ b/tests/test_downloader_handler_httpx.py @@ -3,11 +3,17 @@ from __future__ import annotations import sys -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, ClassVar import pytest from scrapy import Request +from scrapy.core.downloader.handlers._httpx import ( + HAS_HTTP2, + HAS_SOCKS, + HttpxDownloadHandler, +) +from scrapy.exceptions import DownloadFailedError from tests.test_downloader_handlers_http_base import ( TestHttpBase, TestHttpProxyBase, @@ -15,9 +21,11 @@ from tests.test_downloader_handlers_http_base import ( TestHttpsCustomCiphersBase, TestHttpsInvalidDNSIdBase, TestHttpsInvalidDNSPatternBase, + TestHttpsTLSVersionBase, TestHttpsWrongHostnameBase, TestHttpWithCrawlerBase, TestMitmProxyBase, + TestRealWebsiteBase, TestSimpleHttpsBase, ) from tests.utils.decorators import coroutine_test @@ -35,11 +43,6 @@ pytest.importorskip("httpx") class HttpxDownloadHandlerMixin: @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: - # the import will fail if httpx is not installed - from scrapy.core.downloader.handlers._httpx import ( # noqa: PLC0415 - HttpxDownloadHandler, - ) - return HttpxDownloadHandler @property @@ -72,30 +75,40 @@ class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase): assert "DOWNLOAD_BIND_ADDRESS specifies a port (12345)" in caplog.text assert "Ignoring the port" in caplog.text - @coroutine_test - async def test_unsupported_proxy( - self, caplog: pytest.LogCaptureFixture, mockserver: MockServer - ) -> None: - meta = {"proxy": "127.0.0.2"} - request = Request(mockserver.url("/text"), meta=meta) - async with self.get_dh() as download_handler: - response = await download_handler.download_request(request) - assert response.body == b"Works" - assert ( - "The 'proxy' request meta key is not supported by HttpxDownloadHandler" - in caplog.text - ) - class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase): handler_supports_bindaddress_meta = False tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher" @pytest.mark.skip(reason="The check is Twisted-specific") - def test_verify_certs_deprecated(self): + def test_verify_certs_deprecated(self) -> None: # type: ignore[override] pass +@pytest.mark.skipif(not HAS_HTTP2, reason="No HTTP/2 support in HttpxDownloadHandler") +class TestHttp2(TestHttps): + http2 = True + handler_supports_http2_dataloss = False + + default_handler_settings: ClassVar[dict[str, Any]] = { + "HTTPX_HTTP2_ENABLED": True, + } + + @coroutine_test + async def test_protocol(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/host", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.protocol == "HTTP/2" + + @coroutine_test + async def test_data_loss_handling(self, mockserver: MockServer) -> None: + request = Request(mockserver.url("/broken", is_secure=self.is_secure)) + async with self.get_dh() as download_handler: + with pytest.raises(DownloadFailedError): + await download_handler.download_request(request) + + class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase): pass @@ -118,6 +131,10 @@ class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBa pass +class TestHttpsTLSVersion(HttpxDownloadHandlerMixin, TestHttpsTLSVersionBase): + pass + + class TestHttpWithCrawler(HttpxDownloadHandlerMixin, TestHttpWithCrawlerBase): pass @@ -125,23 +142,20 @@ class TestHttpWithCrawler(HttpxDownloadHandlerMixin, TestHttpWithCrawlerBase): class TestHttpsWithCrawler(TestHttpWithCrawler): is_secure = True - @pytest.mark.skip(reason="response.certificate is not implemented") - @coroutine_test - async def test_response_ssl_certificate(self, mockserver: MockServer) -> None: - pass - -@pytest.mark.skip(reason="Proxy support is not implemented yet") class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): - pass + expected_http_proxy_request_body = b"http://example.com/" -@pytest.mark.skip(reason="Proxy support is not implemented yet") -class TestHttpsProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase): +class TestHttpsProxy(TestHttpProxy): is_secure = True -@pytest.mark.skip(reason="Proxy support is not implemented yet") @pytest.mark.requires_mitmproxy class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase): + handler_supports_socks = HAS_SOCKS + + +@pytest.mark.requires_internet +class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase): pass diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index 89cf58c96..79750a136 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sys from typing import TYPE_CHECKING, Any import pytest @@ -17,9 +18,11 @@ from tests.test_downloader_handlers_http_base import ( TestHttpsCustomCiphersBase, TestHttpsInvalidDNSIdBase, TestHttpsInvalidDNSPatternBase, + TestHttpsTLSVersionBase, TestHttpsWrongHostnameBase, TestHttpWithCrawlerBase, TestMitmProxyBase, + TestRealWebsiteBase, TestSimpleHttpsBase, ) @@ -81,6 +84,10 @@ class TestHttpsCustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersB pass +class TestHttpsTLSVersion(HTTP11DownloadHandlerMixin, TestHttpsTLSVersionBase): + pass + + class TestHttpWithCrawler(HTTP11DownloadHandlerMixin, TestHttpWithCrawlerBase): pass @@ -103,3 +110,10 @@ class TestHttpsProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase): class TestMitmProxy(HTTP11DownloadHandlerMixin, TestMitmProxyBase): # not implemented handler_supports_tls_in_tls = False + + +@pytest.mark.requires_internet +class TestRealWebsite(HTTP11DownloadHandlerMixin, TestRealWebsiteBase): + @property + def platform_cert_store_works(self) -> bool: + return sys.platform != "win32" diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 60b276166..5f79a5453 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -2,6 +2,7 @@ from __future__ import annotations +import sys from typing import TYPE_CHECKING, Any import pytest @@ -18,9 +19,11 @@ from tests.test_downloader_handlers_http_base import ( TestHttpsCustomCiphersBase, TestHttpsInvalidDNSIdBase, TestHttpsInvalidDNSPatternBase, + TestHttpsTLSVersionBase, TestHttpsWrongHostnameBase, TestHttpWithCrawlerBase, TestMitmProxyBase, + TestRealWebsiteBase, ) from tests.utils.decorators import coroutine_test @@ -171,6 +174,10 @@ class TestHttp2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase) pass +class TestHttp2TLSVersion(H2DownloadHandlerMixin, TestHttpsTLSVersionBase): + pass + + class TestHttp2WithCrawler(H2DownloadHandlerMixin, TestHttpWithCrawlerBase): is_secure = True @@ -196,3 +203,10 @@ class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): @pytest.mark.requires_mitmproxy class TestMitmProxy(H2DownloadHandlerMixin, TestMitmProxyBase): pass + + +@pytest.mark.requires_internet +class TestRealWebsite(H2DownloadHandlerMixin, TestRealWebsiteBase): + @property + def platform_cert_store_works(self) -> bool: + return sys.platform != "win32" diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 138ddd944..0e1ff07c9 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -6,7 +6,6 @@ import gzip import json import logging import os -import platform import re import sys from abc import ABC, abstractmethod @@ -18,6 +17,7 @@ from typing import TYPE_CHECKING, Any, ClassVar from urllib.parse import urlparse import pytest +from cryptography.x509 import load_der_x509_certificate from twisted.internet.ssl import Certificate from twisted.python.failure import Failure @@ -33,12 +33,13 @@ from scrapy.exceptions import ( UnsupportedURLSchemeError, ) from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse +from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1 from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.misc import build_from_crawler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests import NON_EXISTING_RESOLVABLE -from tests.mockserver.mitm_proxy import MitmProxy, wrong_credentials +from tests.mockserver.mitm_proxy import wrong_credentials from tests.mockserver.proxy_echo import ProxyEchoMockServer from tests.mockserver.simple_https import SimpleMockServer from tests.spiders import ( @@ -72,6 +73,7 @@ class TestHttpBase(ABC): handler_supports_http2_dataloss: bool = True # default headers added by the underlying library that cannot be suppressed always_present_req_headers: ClassVar[frozenset[str]] = frozenset() + default_handler_settings: ClassVar[dict[str, Any]] = {} @property @abstractmethod @@ -82,6 +84,10 @@ class TestHttpBase(ABC): async def get_dh( self, settings_dict: dict[str, Any] | None = None ) -> AsyncGenerator[DownloadHandlerProtocol]: + settings_dict = { + **self.default_handler_settings, + **(settings_dict or {}), + } crawler = get_crawler(DefaultSpider, settings_dict) crawler.spider = crawler._create_spider() dh = build_from_crawler(self.download_handler_cls, crawler) @@ -338,9 +344,7 @@ class TestHttpBase(ABC): @coroutine_test async def test_timeout_download_from_spider_server_hangs( - self, - mockserver: MockServer, - reactor_pytest: str, + self, mockserver: MockServer, reactor_pytest: str ) -> None: if reactor_pytest == "asyncio" and sys.platform == "win32": # https://twistedmatrix.com/trac/ticket/10279 @@ -524,7 +528,7 @@ class TestHttpBase(ABC): await download_handler.download_request(request) assert "download_latency" in request.meta latency = request.meta["download_latency"] - if sys.version_info < (3, 13) and platform.system() == "Windows": + if sys.version_info < (3, 13) and sys.platform == "win32": # time.monotonic() resolution is too low here: # https://docs.python.org/3/whatsnew/3.13.html#time assert latency >= 0 @@ -803,6 +807,25 @@ class TestHttpBase(ABC): "The 'bindaddress' request meta key is not supported by" in caplog.text ) + @coroutine_test + async def test_verbatim_url(self, mockserver: MockServer) -> None: + # Square brackets are encoded by safe_url_string (w3lib). + path = "/uri/items?data[0]=a" + url = mockserver.url(path, is_secure=self.is_secure) + + # Without verbatim_url, the brackets are percent-encoded before the + # request reaches the server. + request = Request(url) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == b"/uri/items?data%5B0%5D=a" + + # With verbatim_url=True the URL is sent to the server as-is. + request = Request(url, meta={"verbatim_url": True}) + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.body == path.encode() + class TestHttpsBase(TestHttpBase): is_secure = True @@ -875,7 +898,7 @@ class TestSimpleHttpsBase(ABC): @pytest.fixture(scope="class") def simple_mockserver(self) -> Generator[SimpleMockServer]: with SimpleMockServer( - self.keyfile, self.certfile, self.cipher_string + self.keyfile, self.certfile, cipher_string=self.cipher_string ) as simple_mockserver: yield simple_mockserver @@ -938,6 +961,143 @@ class TestHttpsCustomCiphersBase(TestSimpleHttpsBase): cipher_string = "CAMELLIA256-SHA" +class TestHttpsTLSVersionBase(ABC): + keyfile = "keys/localhost.key" + certfile = "keys/localhost.crt" + + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + @asynccontextmanager + async def get_dh( + self, client_tls_min: str | None, client_tls_max: str | None + ) -> AsyncGenerator[DownloadHandlerProtocol]: + settings = {} + if client_tls_min is not None: + settings["DOWNLOAD_TLS_MIN_VERSION"] = client_tls_min + if client_tls_max is not None: + settings["DOWNLOAD_TLS_MAX_VERSION"] = client_tls_max + crawler = get_crawler(DefaultSpider, settings_dict=settings) + crawler.spider = crawler._create_spider() + dh = build_from_crawler(self.download_handler_cls, crawler) + try: + yield dh + finally: + await dh.close() + + @pytest.mark.parametrize( + ( + "server_tls_min", + "server_tls_max", + "client_tls_min", + "client_tls_max", + "expect_success", + ), + [ + pytest.param(None, None, None, None, True, id="no-limits"), + pytest.param(None, None, None, "TLSv1.2", True, id="client-max-tls1.2"), + pytest.param(None, None, "TLSv1.3", None, True, id="client-min-tls1.3"), + pytest.param( + "TLSv1.3", + None, + None, + "TLSv1.2", + False, + id="client-max-below-server-min", + ), + pytest.param( + None, + "TLSv1.2", + "TLSv1.3", + None, + False, + id="client-min-above-server-max", + ), + pytest.param(None, "TLSv1.2", None, "TLSv1.2", True, id="both-tls1.2"), + pytest.param("TLSv1.3", None, "TLSv1.3", None, True, id="both-tls1.3"), + pytest.param( + None, + None, + "TLSv1.0", + None, + True, + id="client-min-tls1.0", + marks=pytest.mark.filterwarnings( + r"ignore:ssl\.TLSVersion\.TLSv1 is deprecated:DeprecationWarning" + ), + ), + pytest.param( + "TLSv1.0", + None, + "TLSv1.0", + None, + True, + id="both-min-tls1.0", + marks=pytest.mark.filterwarnings( + r"ignore:ssl\.TLSVersion\.TLSv1 is deprecated:DeprecationWarning" + ), + ), + pytest.param( + "TLSv1.2", + "TLSv1.3", + "TLSv1.2", + "TLSv1.3", + True, + id="both-tls1.2-1.3", + marks=pytest.mark.xfail( + TWISTED_TLS_LIMITS_OFFBY1, + reason="Can't set max to 1.3 on this Twisted version", + strict=True, + ), + ), + ], + ) + @coroutine_test + async def test_download( + self, + server_tls_min: str | None, + server_tls_max: str | None, + client_tls_min: str | None, + client_tls_max: str | None, + expect_success: bool, + ) -> None: + with SimpleMockServer( + self.keyfile, + self.certfile, + tls_min_version=server_tls_min, + tls_max_version=server_tls_max, + ) as simple_mockserver: + url = f"https://localhost:{simple_mockserver.port(is_secure=True)}/file" + request = Request(url) + async with self.get_dh(client_tls_min, client_tls_max) as dh: + if expect_success: + response = await dh.download_request(request) + assert response.body == b"0123456789" + else: + with pytest.raises( + (DownloadConnectionRefusedError, DownloadFailedError) + ): + await dh.download_request(request) + + @coroutine_test + async def test_invalid_min_version_setting(self) -> None: + with pytest.raises( + ValueError, match="Unknown DOWNLOAD_TLS_MIN_VERSION value: invalid" + ): + async with self.get_dh(client_tls_min="invalid", client_tls_max=None): + pass + + @coroutine_test + async def test_invalid_max_version_setting(self) -> None: + with pytest.raises( + ValueError, match="Unknown DOWNLOAD_TLS_MAX_VERSION value: invalid" + ): + async with self.get_dh(client_tls_min=None, client_tls_max="invalid"): + pass + + class TestHttpWithCrawlerBase(ABC): @property @abstractmethod @@ -982,15 +1142,19 @@ class TestHttpWithCrawlerBase(ABC): if not self.is_secure: pytest.skip("Only applies to HTTPS") # copy of TestCrawl.test_response_ssl_certificate() - # the current test implementation can only work for Twisted-based download handlers crawler = get_crawler(SingleRequestSpider, self.settings_dict) url = mockserver.url("/echo?body=test", is_secure=self.is_secure) await crawler.crawl_async(seed=url, mockserver=mockserver) assert isinstance(crawler.spider, SingleRequestSpider) cert = crawler.spider.meta["responses"][0].certificate - assert isinstance(cert, Certificate) - assert cert.getSubject().commonName == b"localhost" - assert cert.getIssuer().commonName == b"localhost" + assert cert is not None + if isinstance(cert, Certificate): # Twisted + assert cert.getSubject().commonName == b"localhost" + assert cert.getIssuer().commonName == b"localhost" + elif isinstance(cert, bytes): # DER bytes + cert_x509 = load_der_x509_certificate(cert) + assert cert_x509.subject.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE" + assert cert_x509.issuer.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE" @coroutine_test async def test_response_ip_address(self, mockserver: MockServer) -> None: @@ -1156,6 +1320,7 @@ class TestHttpProxyBase(ABC): class TestMitmProxyBase(ABC): # whether the handler supports HTTPS proxies with HTTPS destinations handler_supports_tls_in_tls: bool = True + handler_supports_socks: bool = False @property @abstractmethod @@ -1165,13 +1330,10 @@ class TestMitmProxyBase(ABC): @pytest.mark.parametrize( "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] ) + @pytest.mark.usefixtures("mitm_proxy_server") @coroutine_test async def test_http_proxy( - self, - caplog: pytest.LogCaptureFixture, - mockserver: MockServer, - mitm_proxy_server: MitmProxy, - https_dest: bool, + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool ) -> None: """HTTP proxy, HTTP or HTTPS destination.""" crawler = get_crawler(SingleRequestSpider, self.settings_dict) @@ -1186,13 +1348,10 @@ class TestMitmProxyBase(ABC): @pytest.mark.parametrize( "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] ) + @pytest.mark.usefixtures("mitm_proxy_server_https") @coroutine_test async def test_https_proxy( - self, - caplog: pytest.LogCaptureFixture, - mockserver: MockServer, - mitm_proxy_server_https: MitmProxy, - https_dest: bool, + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool ) -> None: """HTTPS proxy, HTTP or HTTPS destination.""" if https_dest and not self.handler_supports_tls_in_tls: @@ -1209,13 +1368,13 @@ class TestMitmProxyBase(ABC): @pytest.mark.parametrize( "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] ) + @pytest.mark.usefixtures("mitm_proxy_server") @coroutine_test async def test_http_proxy_auth_error( self, caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, mockserver: MockServer, - mitm_proxy_server: MitmProxy, https_dest: bool, ) -> None: """HTTP proxy, HTTP or HTTPS destination, wrong proxy creds.""" @@ -1233,13 +1392,10 @@ class TestMitmProxyBase(ABC): @pytest.mark.parametrize( "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] ) + @pytest.mark.usefixtures("mitm_proxy_server") @coroutine_test async def test_dont_leak_proxy_authorization_header( - self, - caplog: pytest.LogCaptureFixture, - mockserver: MockServer, - mitm_proxy_server: MitmProxy, - https_dest: bool, + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool ) -> None: """HTTP proxy, HTTP or HTTPS destination. Check that the auth header is not sent to the destination.""" @@ -1253,6 +1409,49 @@ class TestMitmProxyBase(ABC): echo = json.loads(crawler.spider.meta["responses"][0].text) assert "Proxy-Authorization" not in echo["headers"] + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @pytest.mark.usefixtures("socks5_proxy_server") + @coroutine_test + async def test_download_with_socks_proxy( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer, https_dest: bool + ) -> None: + """SOCKS5 proxy, HTTP or HTTPS destination.""" + if not self.handler_supports_socks: + pytest.skip("SOCKS proxies are not supported") + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + seed=mockserver.url("/status?n=200", is_secure=https_dest) + ) + assert isinstance(crawler.spider, SingleRequestSpider) + self._assert_got_response_code(200, caplog.text) + self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest) + + @pytest.mark.parametrize( + "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"] + ) + @pytest.mark.usefixtures("socks5_proxy_server") + @coroutine_test + async def test_socks_proxy_auth_error( + self, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + mockserver: MockServer, + https_dest: bool, + ) -> None: + if not self.handler_supports_socks: + pytest.skip("SOCKS proxies are not supported") + envvar = "https_proxy" if https_dest else "http_proxy" + monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar])) + crawler = get_crawler(SimpleSpider, self.settings_dict) + with caplog.at_level(logging.DEBUG): + await crawler.crawl_async( + mockserver.url("/status?n=200", is_secure=https_dest) + ) + assert "DownloadConnectionRefusedError" in caplog.text + @staticmethod def _assert_headers(headers: Headers, https_dest: bool) -> None: assert b"X-Via-Mitmproxy" in headers @@ -1266,3 +1465,87 @@ class TestMitmProxyBase(ABC): @staticmethod def _assert_got_auth_exception(log: str) -> None: assert "Proxy Authentication Required" in log or "407" in log + + +class TestRealWebsiteBase(ABC): + @property + @abstractmethod + def download_handler_cls(self) -> type[DownloadHandlerProtocol]: + raise NotImplementedError + + @property + @abstractmethod + def settings_dict(self) -> dict[str, Any] | None: + raise NotImplementedError + + @property + def platform_cert_store_works(self) -> bool: + """Whether valid certificates can be verified. + + Twisted on Windows cannot do that out of the box, see e.g. + https://github.com/twisted/twisted/issues/6371. + """ + return True + + @asynccontextmanager + async def get_dh( + self, settings_dict: dict[str, Any] | None = None + ) -> AsyncGenerator[DownloadHandlerProtocol]: + crawler = get_crawler(DefaultSpider, settings_dict) + crawler.spider = crawler._create_spider() + dh = build_from_crawler(self.download_handler_cls, crawler) + try: + yield dh + finally: + await dh.close() + + @coroutine_test + async def test_download(self) -> None: + request = Request("https://books.toscrape.com/") + async with self.get_dh() as download_handler: + response = await download_handler.download_request(request) + assert response.status == 200 + assert "All products | Books to Scrape - Sandbox" in response.text + + @coroutine_test + async def test_download_with_spider(self) -> None: + crawler = get_crawler(SingleRequestSpider, self.settings_dict) + await maybe_deferred_to_future( + crawler.crawl(seed=Request("https://books.toscrape.com/")) + ) + assert isinstance(crawler.spider, SingleRequestSpider) + failure = crawler.spider.meta.get("failure") + assert failure is None + reason = crawler.spider.meta["close_reason"] + assert reason == "finished" + + @coroutine_test + async def test_verify_certs(self) -> None: + if not self.platform_cert_store_works: + pytest.skip("Cannot verify certificates") + request = Request("https://books.toscrape.com/") + async with self.get_dh( + {"DOWNLOAD_VERIFY_CERTIFICATES": True} + ) as download_handler: + response = await download_handler.download_request(request) + assert response.status == 200 + assert "All products | Books to Scrape - Sandbox" in response.text + + @pytest.mark.parametrize("verify_certs", [True, False]) + @coroutine_test + async def test_tls_logging( + self, caplog: pytest.LogCaptureFixture, verify_certs: bool + ) -> None: + if verify_certs and not self.platform_cert_store_works: + pytest.skip("Cannot verify certificates") + request = Request("https://books.toscrape.com/") + async with self.get_dh( + { + "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True, + "DOWNLOAD_VERIFY_CERTIFICATES": verify_certs, + } + ) as download_handler: + with caplog.at_level("DEBUG"): + response = await download_handler.download_request(request) + assert response.status == 200 + assert "SSL connection to books.toscrape.com using protocol" in caplog.text diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py index 49215329e..225562644 100644 --- a/tests/test_downloadermiddleware_cookies.py +++ b/tests/test_downloadermiddleware_cookies.py @@ -42,7 +42,7 @@ def _cookie_to_set_cookie_value(cookie): def _cookies_to_set_cookie_list(cookies): """Given a group of cookie defined either as a dictionary or as a list of dictionaries (i.e. in a format supported by the cookies parameter of - Request), return the equivalen list of strings that can be associated to a + Request), return the equivalent list of strings that can be associated to a ``Set-Cookie`` header.""" if not cookies: return [] diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 50946899a..56d21a4d2 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -84,6 +84,39 @@ class TestRetry: ) assert self.crawler.stats.get_value("retry/count") == 2 + def test_give_up_log_level_setting(self): + crawler = get_crawler( + DefaultSpider, settings_dict={"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"} + ) + crawler.spider = crawler._create_spider() + mw = RetryMiddleware.from_crawler(crawler) + mw.max_retry_times = 0 + req = Request("http://example.com/503") + rsp = Response("http://example.com/503", body=b"", status=503) + with LogCapture() as log: + assert mw.process_response(req, rsp) is rsp + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", + ) + ) + + def test_give_up_log_level_meta(self): + self.mw.max_retry_times = 0 + req = Request("http://example.com/503", meta={"give_up_log_level": "WARNING"}) + rsp = Response("http://example.com/503", body=b"", status=503) + with LogCapture() as log: + assert self.mw.process_response(req, rsp) is rsp + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable", + ) + ) + def test_twistederrors(self): exceptions = [ ConnectError, @@ -612,6 +645,87 @@ class TestGetRetryRequest: ) ) + def test_give_up_log_level_default(self): + request = Request("https://example.com") + spider = self.get_spider() + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + ) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "ERROR", + f"Gave up retrying {request} (failed 1 times): unspecified", + ) + ) + + def test_give_up_log_level_argument_name(self): + request = Request("https://example.com") + spider = self.get_spider() + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + give_up_log_level="WARNING", + ) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {request} (failed 1 times): unspecified", + ) + ) + + def test_give_up_log_level_argument_number(self): + request = Request("https://example.com") + spider = self.get_spider() + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + give_up_log_level=logging.WARNING, + ) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {request} (failed 1 times): unspecified", + ) + ) + + def test_give_up_log_level_setting(self): + request = Request("https://example.com") + spider = self.get_spider({"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"}) + with LogCapture() as log: + get_retry_request( + request, + spider=spider, + max_retry_times=0, + ) + log.check_present( + ( + "scrapy.downloadermiddlewares.retry", + "WARNING", + f"Gave up retrying {request} (failed 1 times): unspecified", + ) + ) + + def test_give_up_log_level_invalid(self): + request = Request("https://example.com") + spider = self.get_spider() + with pytest.raises(ValueError, match="Invalid give-up log level"): + get_retry_request( + request, + spider=spider, + max_retry_times=0, + give_up_log_level="NOT_A_LEVEL", + ) + def test_custom_stats_key(self): request = Request("https://example.com") spider = self.get_spider() diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index 8b5705487..6cc8c0650 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -263,7 +263,7 @@ class TestRequestSendOrder: @coroutine_test async def test_shared_queues(self): """If SCHEDULER_START_*_QUEUE is falsy, start requests and other - requests share the same queue, i.e. start requests are not priorized + requests share the same queue, i.e. start requests are not prioritized over other requests if their priority matches.""" nums = list(range(1, 14)) response_seconds = 0 diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index cbf568524..27e0c6445 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -41,7 +41,7 @@ from tests.spiders import ItemSpider from tests.utils.decorators import coroutine_test, inline_callbacks_test if TYPE_CHECKING: - from collections.abc import Callable, Iterable + from collections.abc import Awaitable, Callable, Iterable def path_to_url(path: str | Path) -> str: @@ -1312,7 +1312,9 @@ class TestFeedExporterSignals: self.feed_slot_closed_received = True async def run_signaled_feed_exporter( - self, feed_exporter_signal_handler: Callable, feed_slot_signal_handler: Callable + self, + feed_exporter_signal_handler: Callable[[], Awaitable[None] | None], + feed_slot_signal_handler: Callable[[Any], Awaitable[None] | None], ) -> None: crawler = get_crawler(settings_dict=self.settings) feed_exporter = FeedExporter.from_crawler(crawler) diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py index 3b70e896c..d855d0f74 100644 --- a/tests/test_feedexport_batch.py +++ b/tests/test_feedexport_batch.py @@ -29,7 +29,7 @@ if TYPE_CHECKING: from os import PathLike -def build_url(path: str | PathLike) -> str: +def build_url(path: str | PathLike[str]) -> str: path_str = str(path) if path_str[0] != "/": path_str = "/" + path_str diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py index 5102f8e37..11e79775f 100644 --- a/tests/test_feedexport_storages.py +++ b/tests/test_feedexport_storages.py @@ -513,6 +513,34 @@ class TestGCSFeedStorage: client_mock.get_bucket.assert_called_once_with("mybucket") bucket_mock.blob.assert_called_once_with("export.csv") blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + f.close.assert_called_once_with() + + @coroutine_test + async def test_store_closes_file_on_upload_error(self): + try: + from google.cloud.storage import Client # noqa: F401,PLC0415 + except ImportError: + pytest.skip("GCSFeedStorage requires google-cloud-storage") + + uri = "gs://mybucket/export.csv" + project_id = "myproject-123" + acl = "publicRead" + (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage() + blob_mock.upload_from_file.side_effect = OSError("Upload failed") + with mock.patch("google.cloud.storage.Client") as m: + m.return_value = client_mock + + f = mock.Mock() + storage = GCSFeedStorage(uri, project_id, acl) + with pytest.raises(OSError, match="Upload failed"): + await maybe_deferred_to_future(storage.store(f)) + + f.seek.assert_called_once_with(0) + m.assert_called_once_with(project=project_id) + client_mock.get_bucket.assert_called_once_with("mybucket") + bucket_mock.blob.assert_called_once_with("export.csv") + blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl) + f.close.assert_called_once_with() def test_overwrite_default(self): with LogCapture() as log: diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 6072e2f6d..cec5d728b 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -61,7 +61,7 @@ def make_html_body(val: str) -> bytes: class DummySpider(Spider): name = "dummy" - start_urls: list = [] + start_urls = [] def parse(self, response): print(response) diff --git a/tests/test_http_request.py b/tests/test_http_request.py index 4c77bb1ec..fed5dbab7 100644 --- a/tests/test_http_request.py +++ b/tests/test_http_request.py @@ -162,6 +162,19 @@ class TestRequest: r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html") assert r4.url == "http://www.example.org/r%E9sum%E9.html" + def test_url_verbatim(self): + r = self.request_class( + url="http://www.scrapy.org/price/£", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/price/£" + + r = self.request_class( + url="http://www.scrapy.org/blank space", + meta={"verbatim_url": True}, + ) + assert r.url == "http://www.scrapy.org/blank space" + def test_body(self): r1 = self.request_class(url="http://www.example.com/") assert r1.body == b"" diff --git a/tests/test_loader.py b/tests/test_loader.py index 7cfc7ed26..c094d25d8 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -39,7 +39,7 @@ class AttrsNameItem: @dataclasses.dataclass class NameDataClass: - name: list = dataclasses.field(default_factory=list) + name: list[str] = dataclasses.field(default_factory=list) # test item loaders diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index dbe8d85a5..c6a41fa6e 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -383,11 +383,11 @@ class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin): class FilesPipelineTestDataClass: name: str # default fields - file_urls: list = dataclasses.field(default_factory=list) - files: list = dataclasses.field(default_factory=list) + file_urls: list[str] = dataclasses.field(default_factory=list) + files: list[dict[str, str]] = dataclasses.field(default_factory=list) # overridden fields - custom_file_urls: list = dataclasses.field(default_factory=list) - custom_files: list = dataclasses.field(default_factory=list) + custom_file_urls: list[str] = dataclasses.field(default_factory=list) + custom_files: list[dict[str, str]] = dataclasses.field(default_factory=list) class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin): diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py index 199ec5afa..38662348f 100644 --- a/tests/test_pipeline_images.py +++ b/tests/test_pipeline_images.py @@ -314,11 +314,11 @@ class TestImagesPipelineFieldsItem(TestImagesPipelineFieldsMixin): class ImagesPipelineTestDataClass: name: str # default fields - image_urls: list = dataclasses.field(default_factory=list) - images: list = dataclasses.field(default_factory=list) + image_urls: list[str] = dataclasses.field(default_factory=list) + images: list[dict[str, str]] = dataclasses.field(default_factory=list) # overridden fields - custom_image_urls: list = dataclasses.field(default_factory=list) - custom_images: list = dataclasses.field(default_factory=list) + custom_image_urls: list[str] = dataclasses.field(default_factory=list) + custom_images: list[dict[str, str]] = dataclasses.field(default_factory=list) class TestImagesPipelineFieldsDataClass(TestImagesPipelineFieldsMixin): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 8d3977452..0d96e1d88 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -38,8 +38,8 @@ class InjectArgumentsSpiderMiddleware: if request.callback.__name__ == "parse_spider_mw": request.cb_kwargs["from_process_spider_input"] = True - def process_spider_output(self, response, result): - for element in result: + async def process_spider_output(self, response, result): + async for element in result: if ( isinstance(element, Request) and element.callback.__name__ == "parse_spider_mw_2" diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 176fefbbf..db023e1f8 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -41,10 +41,10 @@ class MinimalScheduler: class SimpleScheduler(MinimalScheduler): - def open(self, spider: Spider) -> defer.Deferred: + def open(self, spider: Spider) -> defer.Deferred[str]: return defer.succeed("open") - def close(self, reason: str) -> defer.Deferred: + def close(self, reason: str) -> defer.Deferred[str]: return defer.succeed("close") def __len__(self) -> int: diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py index 1d4c3d487..7b3c52d65 100644 --- a/tests/test_settings/__init__.py +++ b/tests/test_settings/__init__.py @@ -384,15 +384,15 @@ class TestBaseSettings: "TEST_STRING": "a string", "TEST_LIST": [1, 2], "TEST_BOOLEAN": False, - "TEST_BASE": BaseSettings({1: 1, 2: 2}, "project"), - "TEST": BaseSettings({1: 10, 3: 30}, "default"), - "HASNOBASE": BaseSettings({3: 3000}, "default"), + "TEST_BASE": BaseSettings({"foo": 1, "bar": 2}, "project"), + "TEST": BaseSettings({"foo": 10, "baz": 30}, "default"), + "HASNOBASE": BaseSettings({"baz": 3000}, "default"), } ) assert s.copy_to_dict() == { - "HASNOBASE": {3: 3000}, - "TEST": {1: 10, 3: 30}, - "TEST_BASE": {1: 1, 2: 2}, + "HASNOBASE": {"baz": 3000}, + "TEST": {"foo": 10, "baz": 30}, + "TEST_BASE": {"foo": 1, "bar": 2}, "TEST_LIST": [1, 2], "TEST_BOOLEAN": False, "TEST_STRING": "a string", diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py index 05d77912f..c97934b74 100644 --- a/tests/test_spider_crawl.py +++ b/tests/test_spider_crawl.py @@ -4,6 +4,7 @@ import re import warnings from logging import ERROR +import pytest from testfixtures import LogCapture from w3lib.url import safe_url_string @@ -232,6 +233,7 @@ class TestCrawlSpider(TestSpider): "HtmlResponse", ] + @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") def test_follow_links_attribute_population(self): crawler = get_crawler() spider = self.spider_class.from_crawler(crawler, "example.com") diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index bfa60ac9c..a0d296553 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Any, cast from unittest import mock import pytest -from testfixtures import LogCapture from twisted.internet import defer from scrapy.core.spidermw import SpiderMiddlewareManager @@ -63,20 +62,6 @@ class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware): await self._scrape_response() -class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware): - """Invalid return value for process_spider_output method""" - - @coroutine_test - async def test_invalid_process_spider_output(self): - class InvalidProcessSpiderOutputMiddleware: - def process_spider_output(self, response, result): - return 1 - - self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware()) - with pytest.raises(_InvalidOutput): - await self._scrape_response() - - class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): """Invalid return value for process_spider_exception method""" @@ -87,13 +72,15 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware): return 1 class RaiseExceptionProcessSpiderOutputMiddleware: - def process_spider_output(self, response, result): + async def process_spider_output(self, response, result): raise RuntimeError + yield # pylint: disable=unreachable self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) + it = await self._scrape_response() with pytest.raises(_InvalidOutput): - await self._scrape_response() + await collect_asyncgen(it) class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): @@ -106,13 +93,15 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware): return None class RaiseExceptionProcessSpiderOutputMiddleware: - def process_spider_output(self, response, result): + async def process_spider_output(self, response, result): 1 / 0 + yield self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware()) self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware()) + it = await self._scrape_response() with pytest.raises(ZeroDivisionError): - await self._scrape_response() + await collect_asyncgen(it) class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): @@ -121,7 +110,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): Should work for process_spider_output and, when it's supported, process_start. """ - ITEM_TYPE: type | tuple + ITEM_TYPE: type | tuple[type, ...] RESULT_COUNT = 3 # to simplify checks, let everything return 3 objects @staticmethod @@ -155,43 +144,19 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware): self._scrape_func, self.response, self.request ) - async def _test_simple_base( - self, - *mw_classes: type[Any], - downgrade: bool = False, - start_index: int | None = None, - ) -> None: - with LogCapture() as log: - result = await self._get_middleware_result( - *mw_classes, start_index=start_index - ) - assert isinstance(result, Iterable) - result_list = list(result) - assert len(result_list) == self.RESULT_COUNT - assert isinstance(result_list[0], self.ITEM_TYPE) - assert ("downgraded to a non-async" in str(log)) == downgrade - assert ("doesn't support asynchronous spider output" in str(log)) == ( - ProcessSpiderOutputSimpleMiddleware in mw_classes - ) - async def _test_asyncgen_base( self, *mw_classes: type[Any], - downgrade: bool = False, start_index: int | None = None, ) -> None: - with LogCapture() as log: - result = await self._get_middleware_result( - *mw_classes, start_index=start_index - ) + result = await self._get_middleware_result(*mw_classes, start_index=start_index) assert isinstance(result, AsyncIterator) result_list = await collect_asyncgen(result) assert len(result_list) == self.RESULT_COUNT assert isinstance(result_list[0], self.ITEM_TYPE) - assert ("downgraded to a non-async" in str(log)) == downgrade -class ProcessSpiderOutputSimpleMiddleware: +class ProcessSpiderOutputSyncMiddleware: def process_spider_output(self, response, result): yield from result @@ -232,53 +197,36 @@ class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware): """process_spider_output tests for simple callbacks""" ITEM_TYPE = dict - MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware + MW_SYNC = ProcessSpiderOutputSyncMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware @coroutine_test - async def test_simple(self): - """Simple mw""" - await self._test_simple_base(self.MW_SIMPLE) + async def test_sync(self): + """Unsupported sync mw""" + with pytest.raises( + TypeError, match=r"doesn't support asynchronous spider output" + ): + await self._get_middleware_result(self.MW_SYNC) @coroutine_test async def test_asyncgen(self): - """Asyncgen mw; upgrade""" + """Asyncgen mw""" await self._test_asyncgen_base(self.MW_ASYNCGEN) - @coroutine_test - async def test_simple_asyncgen(self): - """Simple mw -> asyncgen mw; upgrade""" - await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE) - - @coroutine_test - async def test_asyncgen_simple(self): - """Asyncgen mw -> simple mw; upgrade then downgrade""" - await self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True) - @coroutine_test async def test_universal(self): """Universal mw""" - await self._test_simple_base(self.MW_UNIVERSAL) - - @coroutine_test - async def test_universal_simple(self): - """Universal mw -> simple mw""" - await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL) - - @coroutine_test - async def test_simple_universal(self): - """Simple mw -> universal mw""" - await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE) + await self._test_asyncgen_base(self.MW_UNIVERSAL) @coroutine_test async def test_universal_asyncgen(self): - """Universal mw -> asyncgen mw; upgrade""" + """Universal mw -> asyncgen mw""" await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL) @coroutine_test async def test_asyncgen_universal(self): - """Asyncgen mw -> universal mw; upgrade""" + """Asyncgen mw -> universal mw""" await self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN) @@ -289,31 +237,6 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple): for item in super()._callback(): yield item - @coroutine_test - async def test_simple(self): - """Simple mw; downgrade""" - await self._test_simple_base(self.MW_SIMPLE, downgrade=True) - - @coroutine_test - async def test_simple_asyncgen(self): - """Simple mw -> asyncgen mw; downgrade then upgrade""" - await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True) - - @coroutine_test - async def test_universal(self): - """Universal mw""" - await self._test_asyncgen_base(self.MW_UNIVERSAL) - - @coroutine_test - async def test_universal_simple(self): - """Universal mw -> simple mw; downgrade""" - await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True) - - @coroutine_test - async def test_simple_universal(self): - """Simple mw -> universal mw; downgrade""" - await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True) - class ProcessSpiderOutputNonIterableMiddleware: def process_spider_output(self, response, result): @@ -325,24 +248,6 @@ class ProcessSpiderOutputCoroutineMiddleware: return result -class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): - @coroutine_test - async def test_non_iterable(self): - with pytest.raises( - _InvalidOutput, - match=r"\.process_spider_output must return an iterable, got ", - ): - await self._get_middleware_result(ProcessSpiderOutputNonIterableMiddleware) - - @coroutine_test - async def test_coroutine(self): - with pytest.raises( - _InvalidOutput, - match=r"\.process_spider_output must be an asynchronous generator", - ): - await self._get_middleware_result(ProcessSpiderOutputCoroutineMiddleware) - - class ProcessStartSimpleMiddleware: async def process_start(self, start): async for item_or_request in start: @@ -415,11 +320,11 @@ class TestUniversalMiddlewareManager: return SpiderMiddlewareManager.from_crawler(crawler) def test_simple_mw(self, mwman: SpiderMiddlewareManager) -> None: - mw = ProcessSpiderOutputSimpleMiddleware() - mwman._add_middleware(mw) - assert ( - mwman.methods["process_spider_output"][0] == mw.process_spider_output # pylint: disable=comparison-with-callable - ) + mw = ProcessSpiderOutputSyncMiddleware() + with pytest.raises( + TypeError, match=r"doesn't support asynchronous spider output" + ): + mwman._add_middleware(mw) def test_async_mw(self, mwman: SpiderMiddlewareManager) -> None: mw = ProcessSpiderOutputAsyncGenMiddleware() @@ -431,9 +336,8 @@ class TestUniversalMiddlewareManager: def test_universal_mw(self, mwman: SpiderMiddlewareManager) -> None: mw = ProcessSpiderOutputUniversalMiddleware() mwman._add_middleware(mw) - assert mwman.methods["process_spider_output"][0] == ( - mw.process_spider_output, - mw.process_spider_output_async, + assert ( + mwman.methods["process_spider_output"][0] == mw.process_spider_output_async # pylint: disable=comparison-with-callable ) def test_universal_mw_no_sync( @@ -441,7 +345,7 @@ class TestUniversalMiddlewareManager: ) -> None: mwman._add_middleware(UniversalMiddlewareNoSync()) assert ( - "UniversalMiddlewareNoSync has process_spider_output_async" + "UniversalMiddlewareNoSync has process_spider_output_async()" " without process_spider_output" in caplog.text ) assert mwman.methods["process_spider_output"][0] is None @@ -465,7 +369,7 @@ class TestUniversalMiddlewareManager: mwman._add_middleware(UniversalMiddlewareBothAsync()) assert ( "UniversalMiddlewareBothAsync.process_spider_output " - "is an async generator function while process_spider_output_async exists" + "is an async generator function while process_spider_output_async() exists" in caplog.text ) assert mwman.methods["process_spider_output"][0] is None @@ -473,7 +377,6 @@ class TestUniversalMiddlewareManager: class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict - MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware @@ -488,66 +391,22 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware): self._scrape_func, self.response, self.request ) - @coroutine_test - async def test_just_builtin(self): - await self._test_simple_base() - - @coroutine_test - async def test_builtin_simple(self): - await self._test_simple_base(self.MW_SIMPLE, start_index=1000) - - @coroutine_test - async def test_builtin_async(self): - """Upgrade""" - await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) - - @coroutine_test - async def test_builtin_universal(self): - await self._test_simple_base(self.MW_UNIVERSAL, start_index=1000) - - @coroutine_test - async def test_simple_builtin(self): - await self._test_simple_base(self.MW_SIMPLE) - - @coroutine_test - async def test_async_builtin(self): - """Upgrade""" - await self._test_asyncgen_base(self.MW_ASYNCGEN) - - @coroutine_test - async def test_universal_builtin(self): - await self._test_simple_base(self.MW_UNIVERSAL) - - -class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): - async def _callback(self) -> Any: - for item in super()._callback(): - yield item - @coroutine_test async def test_just_builtin(self): await self._test_asyncgen_base() - @coroutine_test - async def test_builtin_simple(self): - """Downgrade""" - await self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000) - @coroutine_test async def test_builtin_async(self): + """Upgrade""" await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000) @coroutine_test async def test_builtin_universal(self): await self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000) - @coroutine_test - async def test_simple_builtin(self): - """Downgrade""" - await self._test_simple_base(self.MW_SIMPLE, downgrade=True) - @coroutine_test async def test_async_builtin(self): + """Upgrade""" await self._test_asyncgen_base(self.MW_ASYNCGEN) @coroutine_test @@ -555,9 +414,14 @@ class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): await self._test_asyncgen_base(self.MW_UNIVERSAL) +class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple): + async def _callback(self) -> Any: + for item in super()._callback(): + yield item + + class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): ITEM_TYPE = dict - MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware @@ -576,18 +440,13 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): @coroutine_test async def test_exc_simple(self): """Simple exc mw""" - await self._test_simple_base(self.MW_EXC_SIMPLE) + await self._test_asyncgen_base(self.MW_EXC_SIMPLE) @coroutine_test async def test_exc_async(self): """Async exc mw""" await self._test_asyncgen_base(self.MW_EXC_ASYNCGEN) - @coroutine_test - async def test_exc_simple_simple(self): - """Simple exc mw -> simple output mw""" - await self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE) - @coroutine_test async def test_exc_async_async(self): """Async exc mw -> async output mw""" @@ -598,25 +457,27 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware): """Simple exc mw -> async output mw; upgrade""" await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE) - @coroutine_test - async def test_exc_async_simple(self): - """Async exc mw -> simple output mw; cannot work as downgrading is not supported""" - await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN) - class TestDeprecatedSpiderArg(TestSpiderMiddleware): @coroutine_test async def test_deprecated_mw_spider_arg(self): - class DeprecatedSpiderArgMiddleware: + class DeprecatedSpiderArgMiddleware1: def process_spider_input(self, response, spider): return None - def process_spider_output(self, response, result, spider): + async def process_spider_output(self, response, result, spider): 1 / 0 + yield + class DeprecatedSpiderArgMiddleware2: def process_spider_exception(self, response, exception, spider): return [] + with pytest.warns( + ScrapyDeprecationWarning, + match=r"process_spider_exception\(\) requires a spider argument", + ): + self.mwman._add_middleware(DeprecatedSpiderArgMiddleware2()) with ( pytest.warns( ScrapyDeprecationWarning, @@ -626,13 +487,10 @@ class TestDeprecatedSpiderArg(TestSpiderMiddleware): ScrapyDeprecationWarning, match=r"process_spider_output\(\) requires a spider argument", ), - pytest.warns( - ScrapyDeprecationWarning, - match=r"process_spider_exception\(\) requires a spider argument", - ), ): - self.mwman._add_middleware(DeprecatedSpiderArgMiddleware()) - await self._scrape_response() + self.mwman._add_middleware(DeprecatedSpiderArgMiddleware1()) + it = await self._scrape_response() + await collect_asyncgen(it) @coroutine_test async def test_deprecated_mwman_spider_arg(self): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index d5ad59346..6a054ded5 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,17 +1,19 @@ from __future__ import annotations import logging +from typing import TYPE_CHECKING import pytest -from testfixtures import LogCapture from scrapy.http import Request, Response from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler -from tests.mockserver.http import MockServer from tests.spiders import MockServerSpider -from tests.utils.decorators import inline_callbacks_test +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from tests.mockserver.http import MockServer class _HttpErrorSpider(MockServerSpider): @@ -192,65 +194,66 @@ class TestHttpErrorMiddlewareHandleAll: class TestHttpErrorMiddlewareIntegrational: - @classmethod - def setup_class(cls): - cls.mockserver = MockServer() - cls.mockserver.__enter__() - - @classmethod - def teardown_class(cls): - cls.mockserver.__exit__(None, None, None) - - @inline_callbacks_test - def test_middleware_works(self): + @coroutine_test + async def test_middleware_works(self, mockserver: MockServer) -> None: crawler = get_crawler(_HttpErrorSpider) - yield crawler.crawl(mockserver=self.mockserver) + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, _HttpErrorSpider) assert not crawler.spider.skipped assert crawler.spider.parsed == {"200"} assert crawler.spider.failed == {"404", "402", "500"} + assert crawler.stats get_value = crawler.stats.get_value assert get_value("httperror/response_ignored_count") == 3 assert get_value("httperror/response_ignored_status_count/404") == 1 assert get_value("httperror/response_ignored_status_count/402") == 1 assert get_value("httperror/response_ignored_status_count/500") == 1 - @inline_callbacks_test - def test_logging(self): + @coroutine_test + async def test_logging( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: crawler = get_crawler(_HttpErrorSpider) - with LogCapture() as log: - yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402}) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver, bypass_status_codes={402}) + assert isinstance(crawler.spider, _HttpErrorSpider) assert crawler.spider.parsed == {"200", "402"} assert crawler.spider.skipped == {"402"} assert crawler.spider.failed == {"404", "500"} - assert "Ignoring response <404" in str(log) - assert "Ignoring response <500" in str(log) - assert "Ignoring response <200" not in str(log) - assert "Ignoring response <402" not in str(log) + assert "Ignoring response <404" in caplog.text + assert "Ignoring response <500" in caplog.text + assert "Ignoring response <200" not in caplog.text + assert "Ignoring response <402" not in caplog.text - @inline_callbacks_test - def test_logging_level(self): + @coroutine_test + async def test_logging_level( + self, caplog: pytest.LogCaptureFixture, mockserver: MockServer + ) -> None: # HttpError logs ignored responses with level INFO crawler = get_crawler(_HttpErrorSpider) - with LogCapture(level=logging.INFO) as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.INFO): + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, _HttpErrorSpider) assert crawler.spider.parsed == {"200"} assert crawler.spider.failed == {"404", "402", "500"} - assert "Ignoring response <402" in str(log) - assert "Ignoring response <404" in str(log) - assert "Ignoring response <500" in str(log) - assert "Ignoring response <200" not in str(log) + assert "Ignoring response <402" in caplog.text + assert "Ignoring response <404" in caplog.text + assert "Ignoring response <500" in caplog.text + assert "Ignoring response <200" not in caplog.text # with level WARNING, we shouldn't capture anything from HttpError + caplog.clear() crawler = get_crawler(_HttpErrorSpider) - with LogCapture(level=logging.WARNING) as log: - yield crawler.crawl(mockserver=self.mockserver) + with caplog.at_level(logging.WARNING): + await crawler.crawl_async(mockserver=mockserver) + assert isinstance(crawler.spider, _HttpErrorSpider) assert crawler.spider.parsed == {"200"} assert crawler.spider.failed == {"404", "402", "500"} - assert "Ignoring response <402" not in str(log) - assert "Ignoring response <404" not in str(log) - assert "Ignoring response <500" not in str(log) - assert "Ignoring response <200" not in str(log) + assert "Ignoring response <402" not in caplog.text + assert "Ignoring response <404" not in caplog.text + assert "Ignoring response <500" not in caplog.text + assert "Ignoring response <200" not in caplog.text diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 8cf08dd94..afdfaa322 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -169,8 +169,8 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS # ================================================================================ # (4) exceptions from a middleware process_spider_output method (generator) class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware): - def process_spider_output(self, response, result): - for r in result: + async def process_spider_output(self, response, result): + async for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r @@ -182,8 +182,8 @@ class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware): class GeneratorFailMiddleware(_BaseSpiderMiddleware): - def process_spider_output(self, response, result): - for r in result: + async def process_spider_output(self, response, result): + async for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r raise LookupError @@ -201,8 +201,8 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware): class GeneratorRecoverMiddleware(_BaseSpiderMiddleware): - def process_spider_output(self, response, result): - for r in result: + async def process_spider_output(self, response, result): + async for r in result: r["processed"].append(f"{self.__class__.__name__}.process_spider_output") yield r @@ -237,86 +237,6 @@ class GeneratorOutputChainSpider(Spider): yield {"processed": ["parse-second-item"]} -# ================================================================================ -# (5) exceptions from a middleware process_spider_output method (not generator) - - -class _NotGeneratorDoNothingMiddleware(_BaseSpiderMiddleware): - def process_spider_output(self, response, result): - out = [] - for r in result: - r["processed"].append(f"{self.__class__.__name__}.process_spider_output") - out.append(r) - return out - - def process_spider_exception(self, response, exception): - method = f"{self.__class__.__name__}.process_spider_exception" - self.crawler.spider.logger.info( - "%s: %s caught", method, exception.__class__.__name__ - ) - - -class NotGeneratorFailMiddleware(_BaseSpiderMiddleware): - def process_spider_output(self, response, result): - out = [] - for r in result: - r["processed"].append(f"{self.__class__.__name__}.process_spider_output") - out.append(r) - raise ReferenceError - - def process_spider_exception(self, response, exception): - method = f"{self.__class__.__name__}.process_spider_exception" - self.crawler.spider.logger.info( - "%s: %s caught", method, exception.__class__.__name__ - ) - return [{"processed": [method]}] - - -class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddleware): - pass - - -class NotGeneratorRecoverMiddleware(_BaseSpiderMiddleware): - def process_spider_output(self, response, result): - out = [] - for r in result: - r["processed"].append(f"{self.__class__.__name__}.process_spider_output") - out.append(r) - return out - - def process_spider_exception(self, response, exception): - method = f"{self.__class__.__name__}.process_spider_exception" - self.crawler.spider.logger.info( - "%s: %s caught", method, exception.__class__.__name__ - ) - return [{"processed": [method]}] - - -class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware): - pass - - -class NotGeneratorOutputChainSpider(Spider): - name = "NotGeneratorOutputChainSpider" - custom_settings = { - "SPIDER_MIDDLEWARES": { - NotGeneratorFailMiddleware: 10, - NotGeneratorDoNothingAfterFailureMiddleware: 8, - NotGeneratorRecoverMiddleware: 5, - NotGeneratorDoNothingAfterRecoveryMiddleware: 3, - }, - } - - async def start(self): - yield Request(self.mockserver.url("/status?n=200")) - - def parse(self, response): - return [ - {"processed": ["parse-first-item"]}, - {"processed": ["parse-second-item"]}, - ] - - # ================================================================================ class TestSpiderMiddleware: mockserver: MockServer @@ -481,41 +401,3 @@ class TestSpiderMiddleware: assert str(item_from_callback) in str(log4) assert str(item_recovered) in str(log4) assert "parse-second-item" not in str(log4) - - @coroutine_test - async def test_not_a_generator_output_chain(self): - """ - (5) An exception from a middleware's process_spider_output method should be sent - to the process_spider_exception method from the next middleware in the chain. - The result of the recovery by the process_spider_exception method should be handled - by the process_spider_output method from the next middleware. - The final item count should be 1 (from the process_spider_exception chain, the items - from the spider callback are lost) - """ - log5 = await self.crawl_log(NotGeneratorOutputChainSpider) - assert "'item_scraped_count': 1" in str(log5) - assert ( - "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught" - in str(log5) - ) - assert ( - "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught" - in str(log5) - ) - assert ( - "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught" - not in str(log5) - ) - assert ( - "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught" - not in str(log5) - ) - item_recovered = { - "processed": [ - "NotGeneratorRecoverMiddleware.process_spider_exception", - "NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output", - ] - } - assert str(item_recovered) in str(log5) - assert "parse-first-item" not in str(log5) - assert "parse-second-item" not in str(log5) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index c58b02ca5..f573993a1 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -2,6 +2,7 @@ import copy import warnings from abc import ABC, abstractmethod from collections.abc import Iterator, Mapping, MutableMapping +from typing import Any import pytest @@ -20,7 +21,7 @@ from scrapy.utils.python import garbage_collect class TestCaseInsensitiveDictBase(ABC): @property @abstractmethod - def dict_class(self) -> type[MutableMapping]: + def dict_class(self) -> type[MutableMapping[str, Any]]: raise NotImplementedError def test_init_dict(self): @@ -206,7 +207,7 @@ class TestCaseInsensitiveDictBase(ABC): class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase): - dict_class = CaseInsensitiveDict + dict_class = CaseInsensitiveDict # type: ignore[assignment] def test_repr(self): d1 = self.dict_class({"foo": "bar"}) diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py index f40e424ff..8e5020022 100644 --- a/tests/test_utils_log.py +++ b/tests/test_utils_log.py @@ -42,33 +42,29 @@ class TestFailureToExcInfo: class TestTopLevelFormatter: - def setup_method(self): - self.handler = LogCapture() - self.handler.addFilter(TopLevelFormatter(["test"])) - - def test_top_level_logger(self): + def test_top_level_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("test") - with self.handler as log: - logger.warning("test log msg") - log.check(("test", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples - def test_children_logger(self): + def test_children_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("test.test1") - with self.handler as log: - logger.warning("test log msg") - log.check(("test", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples - def test_overlapping_name_logger(self): + def test_overlapping_name_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("test2") - with self.handler as log: - logger.warning("test log msg") - log.check(("test2", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("test2", logging.WARNING, "test log msg") in caplog.record_tuples - def test_different_name_logger(self): + def test_different_name_logger(self, caplog: pytest.LogCaptureFixture) -> None: + caplog.handler.addFilter(TopLevelFormatter(["test"])) logger = logging.getLogger("different") - with self.handler as log: - logger.warning("test log msg") - log.check(("different", "WARNING", "test log msg")) + logger.warning("test log msg") + assert ("different", logging.WARNING, "test log msg") in caplog.record_tuples class TestLogCounterHandler: @@ -144,7 +140,9 @@ class TestStreamLogger: ], ) def test_spider_logger_adapter_process( - base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict + base_extra: Mapping[str, Any], + log_extra: MutableMapping[str, Any], + expected_extra: dict[str, Any], ) -> None: logger = logging.getLogger("test") spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 8dfac5f02..e8fe45749 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -12,7 +12,6 @@ from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import aiter_errback from scrapy.utils.python import ( MutableAsyncChain, - MutableChain, binary_is_text, get_func_args, memoizemethod_noargs, @@ -30,16 +29,6 @@ _KT = TypeVar("_KT") _VT = TypeVar("_VT") -def test_mutablechain(): - m = MutableChain(range(2), [2, 3], (4, 5)) - m.extend(range(6, 7)) - m.extend([7, 8]) - m.extend([9, 10], (11, 12)) - assert next(m) == 0 - assert m.__next__() == 1 - assert list(m) == list(range(2, 13)) - - class TestMutableAsyncChain: @staticmethod async def g1(): diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py index b25129c87..55c46059e 100644 --- a/tests/test_utils_request.py +++ b/tests/test_utils_request.py @@ -2,6 +2,7 @@ from __future__ import annotations import json from hashlib import sha1 +from typing import Any from weakref import WeakKeyDictionary import pytest @@ -56,13 +57,17 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None: class TestFingerprint: - function: staticmethod = staticmethod(fingerprint) + function: staticmethod[[Request], bytes] = staticmethod(fingerprint) cache: ( - WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]] - | WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]] + WeakKeyDictionary[ + Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes] + ] + | WeakKeyDictionary[ + Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], str] + ] ) = _fingerprint_cache - default_cache_key = (None, False) - known_hashes: tuple[tuple[Request, bytes | str, dict], ...] = ( + default_cache_key = (None, False, False) + known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = ( ( Request("http://example.org"), b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD", @@ -191,6 +196,32 @@ class TestFingerprint: assert self.function(r2) != self.function(r2, keep_fragments=True) assert self.function(r1) != self.function(r2, keep_fragments=True) + def test_verbatim_url(self): + # verbatim_url requests skip URL canonicalization + r1 = Request( + "http://www.example.com/query?a=1&b=2", meta={"verbatim_url": True} + ) + r2 = Request( + "http://www.example.com/query?b=2&a=1", meta={"verbatim_url": True} + ) + assert self.function(r1) != self.function(r2) + + # without verbatim_url, canonicalization makes query-param order irrelevant + r3 = Request("http://www.example.com/query?a=1&b=2") + r4 = Request("http://www.example.com/query?b=2&a=1") + assert self.function(r3) == self.function(r4) + + # with verbatim_url, the fragment is always kept in the fingerprint + r5 = Request( + "http://www.example.com/test#fragment", meta={"verbatim_url": True} + ) + r6 = Request("http://www.example.com/test", meta={"verbatim_url": True}) + assert self.function(r5) != self.function(r6) + + # keep_fragments parameter is ignored for verbatim_url requests + assert self.function(r5) == self.function(r5, keep_fragments=True) + assert self.function(r5) == self.function(r5, keep_fragments=False) + def test_method_and_body(self): r1 = Request("http://www.example.com") r2 = Request("http://www.example.com", method="POST") diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py index 381a9c3ff..e02bdfb69 100644 --- a/tests/test_utils_response.py +++ b/tests/test_utils_response.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse import pytest -from scrapy.http import HtmlResponse, Response +from scrapy.http import HtmlResponse, Response, TextResponse from scrapy.utils.python import to_bytes from scrapy.utils.response import ( _remove_html_comments, @@ -15,6 +15,13 @@ from scrapy.utils.response import ( ) +def _read_browser_output(burl: str): + path = urlparse(burl).path + if not path or not Path(path).exists(): + path = burl.replace("file://", "") + return Path(path).read_bytes() + + def test_open_in_browser(): url = "http:///www.example.com/some/page.html" body = ( @@ -22,10 +29,7 @@ def test_open_in_browser(): ) def browser_open(burl: str) -> bool: - path = urlparse(burl).path - if not path or not Path(path).exists(): - path = burl.replace("file://", "") - bbody = Path(path).read_bytes() + bbody = _read_browser_output(burl) assert b'' in bbody return True @@ -169,10 +173,7 @@ def test_inject_base_url(body: bytes) -> None: url = "http://www.example.com" def check_base_url(burl): - path = urlparse(burl).path - if not path or not Path(path).exists(): - path = burl.replace("file://", "") - bbody = Path(path).read_bytes() + bbody = _read_browser_output(burl) assert bbody.count(b'>') == 1 assert b"ccd", b"acd"), (b"ad", b"ad"), + (b"a -->b", b"a -->b"), + (b"real", b"real"), ], ) def test_remove_html_comments(input_body, output_body): assert _remove_html_comments(input_body) == output_body + + +def test_open_in_browser_preserves_html_comments(): + url = "http://www.example.com" + body = ( + b"" + b"" + b"Real" + b"content" + b"" + ) + + def check(burl): + bbody = _read_browser_output(burl) + assert b"" in bbody + return True + + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_does_not_inject_base_when_present(): + url = "http://www.example.com" + body = ( + b"" + b'T' + b"hi" + b"" + ) + + def check(burl): + bbody = _read_browser_output(burl) + assert b'' not in bbody + assert b'' in bbody + return True + + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_injects_base_when_only_in_comment(): + url = "http://www.example.com" + body = ( + b"" + b"" + b"Real" + b"content" + b"" + ) + + def check(burl): + bbody = _read_browser_output(burl) + assert b'' in bbody + return True + + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_injects_base_at_real_head_not_commented_head(): + url = "http://www.example.com" + body = ( + b"" + b"" + b"Actual" + b"hello" + b"" + ) + + def check(burl): + bbody = _read_browser_output(burl) + assert bbody.count(b'') == 1 + base_pos = bbody.find(b'') + title_pos = bbody.find(b"Actual") + assert base_pos < title_pos + return True + + response = HtmlResponse(url, body=body) + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_text_response_uses_txt_extension(): + response = TextResponse("http://www.example.com", body=b"plain text content") + + def check(burl): + assert burl.endswith(".txt") + return True + + assert open_in_browser(response, _openfunc=check) + + +def test_open_in_browser_raises_for_unsupported_response_type(): + response = Response("http://www.example.com", body=b"binary") + with pytest.raises(TypeError): + open_in_browser(response, _openfunc=lambda _: True) diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.py index 1faa2487a..b2ba013f1 100644 --- a/tests/test_zz_resources.py +++ b/tests/test_zz_resources.py @@ -15,7 +15,8 @@ from tests.utils.decorators import coroutine_test def test_counter_handler() -> None: """Test that ``LogCounterHandler`` is always properly removed. - It's added in ``Crawler.crawl{,_async}()`` and removed on engine_stopped. + It's added in ``LogCount.spider_opened()`` and removed in + ``LogCount.spider_closed()``. """ c = sum(1 for h in logging.root.handlers if isinstance(h, LogCounterHandler)) assert c == 0 @@ -26,7 +27,7 @@ def test_stderr_log_handler() -> None: It's added in ``configure_logging()``, called by ``{Async,}CrawlerProcess`` (without ``install_root_handler=False``). It can be removed with - ``_uninstall_scrapy_root_handler()`` if installing it was really neeeded. + ``_uninstall_scrapy_root_handler()`` if installing it was really needed. """ c = sum(1 for h in logging.root.handlers if type(h) is logging.StreamHandler) # pylint: disable=unidiomatic-typecheck assert c == 0 diff --git a/tox.ini b/tox.ini index 72686d3d0..50ac7aa86 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,31 @@ [tox] requires = sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6 -envlist = pre-commit,pylint,typing,py,docs +envlist = + pre-commit + pylint + mypy + mypy-tests + twinecheck + docs + docs-tests + docs-links + docs-coverage + min + min-extra-deps + min-default-reactor + min-no-reactor + min-botocore + min-pypy3 + py{310,311,312,313,314} + extra-deps + default-reactor + no-reactor + no-reactor-extra-deps + botocore + mitmproxy + pypy3 + pypy3-extra-deps minversion = 1.7.0 [test-requirements] @@ -41,16 +65,16 @@ download = true commands = pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 scrapy tests --doctest-modules} -[testenv:typing] +[testenv:mypy] basepython = python3.10 deps = - mypy==1.20.2 + mypy==2.1.0 typing-extensions==4.15.0 Pillow==12.2.0 Protego==0.6.0 Twisted==26.4.0 attrs==26.1.0 - boto3-stubs[s3]==1.43.2 + boto3-stubs[s3]==1.43.9 botocore-stubs==1.42.41 h2==4.3.0 httpx==0.28.1 @@ -58,23 +82,24 @@ deps = ptpython==3.0.32 # newer ones require newer Python ipython==8.39.0 - pyOpenSSL==26.1.0 + pyOpenSSL==26.2.0 pytest==9.0.3 - types-Pygments==2.20.0.20260408 + socksio==1.0.0 + types-Pygments==2.20.0.20260508 types-defusedxml==0.7.0.20260504 types-lxml==2026.2.16 - types-pexpect==4.9.0.20260408 + types-pexpect==4.9.0.20260508 uvloop==0.22.1 w3lib==2.4.1 zstandard==0.25.0 commands = mypy {posargs:scrapy tests} -[testenv:typing-tests] +[testenv:mypy-tests] basepython = python3.10 deps = {[test-requirements]deps} - {[testenv:typing]deps} + {[testenv:mypy]deps} pytest-mypy-testing==0.2.0 commands = pytest {posargs:tests_typing} @@ -104,7 +129,7 @@ commands = python -m build --sdist twine check dist/* -[pinned] +[min] basepython = python3.10 deps = # pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0 @@ -113,7 +138,7 @@ deps = Twisted==21.7.0 cryptography==37.0.0 cssselect==0.9.1 - httpx==0.26.0 + httpx==0.27.1 itemadapter==0.1.0 lxml==4.6.4 parsel==1.5.0 @@ -124,18 +149,18 @@ deps = zope.interface==5.1.0 {[test-requirements]deps} setenv = - _SCRAPY_PINNED=true + _SCRAPY_MIN=true commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=pinned.junit.xml -o junit_family=legacy --durations=10 scrapy tests} + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=min.junit.xml -o junit_family=legacy --durations=10 scrapy tests} -[testenv:pinned] -basepython = {[pinned]basepython} +[testenv:min] +basepython = {[min]basepython} deps = - {[pinned]deps} + {[min]deps} PyDispatcher==2.0.5 setenv = - {[pinned]setenv} -commands = {[pinned]commands} + {[min]setenv} +commands = {[min]commands} [testenv:extra-deps] basepython = python3 @@ -148,16 +173,16 @@ deps = brotli >= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests brotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests google-cloud-storage - httpx + httpx[http2,socks] ipython robotexclusionrulesparser uvloop; platform_system != "Windows" and implementation_name != "pypy" zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests -[testenv:extra-deps-pinned] -basepython = {[pinned]basepython} +[testenv:min-extra-deps] +basepython = {[min]basepython} deps = - {[pinned]deps} + {[min]deps} Pillow==8.3.2 Twisted[http2]==21.7.0 boto3==1.20.0 @@ -165,25 +190,25 @@ deps = brotli==1.2.0; implementation_name != "pypy" brotlicffi==1.2.0.0; implementation_name == "pypy" google-cloud-storage==1.29.0 - httpx==0.26.0 + httpx==0.27.1 ipython==7.1.0 robotexclusionrulesparser==1.6.2 uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy" zstandard==0.16.0; implementation_name != "pypy" setenv = - {[pinned]setenv} -commands = {[pinned]commands} + {[min]setenv} +commands = {[min]commands} [testenv:default-reactor] commands = {[testenv]commands} --reactor=default -[testenv:default-reactor-pinned] -basepython = {[pinned]basepython} -deps = {[testenv:pinned]deps} -commands = {[pinned]commands} --reactor=default +[testenv:min-default-reactor] +basepython = {[min]basepython} +deps = {[testenv:min]deps} +commands = {[min]commands} --reactor=default setenv = - {[pinned]setenv} + {[min]setenv} [testenv:no-reactor] deps = @@ -199,14 +224,14 @@ deps = commands = {[testenv]commands} -p no:twisted --reactor=none -[testenv:no-reactor-pinned] -basepython = {[pinned]basepython} +[testenv:min-no-reactor] +basepython = {[min]basepython} deps = - {[testenv:pinned]deps} + {[testenv:min]deps} pytest-asyncio -commands = {[pinned]commands} -p no:twisted --reactor=none +commands = {[min]commands} -p no:twisted --reactor=none setenv = - {[pinned]setenv} + {[min]setenv} [testenv:pypy3] basepython = pypy3 @@ -220,7 +245,7 @@ deps = {[testenv:extra-deps]deps} commands = {[testenv:pypy3]commands} -[testenv:pypy3-pinned] +[testenv:min-pypy3] basepython = pypy3.11 deps = PyPyDispatcher==2.1.0 @@ -236,13 +261,18 @@ deps = pyOpenSSL==24.3.0 queuelib==1.4.2 service_identity==23.1.0 - w3lib==1.20.0 + # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses + # an inline flag placement that Python 3.11 treats as an error: global + # flags not at the start of the expression. w3lib 1.18 stopped encoding [] + # in URLs until 2.1.0 brought that behavior back. Tests for verbatim_url + # rely on that encoding. + w3lib==2.1.0 zope.interface==5.1.0 commands = ; disabling coverage pytest {posargs:--durations=10 scrapy tests} setenv = - {[pinned]setenv} + {[min]setenv} [testenv:docs-tests] changedir = docs @@ -277,17 +307,17 @@ deps = commands = pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_botocore -[testenv:botocore-pinned] -basepython = {[pinned]basepython} +[testenv:min-botocore] +basepython = {[min]basepython} deps = - {[pinned]deps} + {[min]deps} botocore==1.13.45 # botocore 1.13.45 requires urllib3>=1.20,<1.26; requests 2.33.0 requires urllib3>=1.26,<3 requests<2.33.0 setenv = - {[pinned]setenv} + {[min]setenv} commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore-pinned.junit.xml -o junit_family=legacy} -m requires_botocore + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore # Run proxy tests that use mitmproxy in a separate env to avoid installing @@ -299,5 +329,6 @@ deps = {[testenv]deps} # mitmproxy does not support PyPy mitmproxy; implementation_name != "pypy" + httpx[http2,socks] commands = - pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_mitmproxy + pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=mitmproxy.junit.xml -o junit_family=legacy} -m requires_mitmproxy