diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index c4d63e22b..e09522bb5 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -80,9 +80,6 @@ jobs: - python-version: "3.14" env: TOXENV: botocore - - python-version: "3.14" - env: - TOXENV: mitmproxy steps: - uses: actions/checkout@v6 @@ -98,6 +95,9 @@ jobs: sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev + - name: Install mitmproxy + run: pipx install mitmproxy + - name: Run tests env: ${{ matrix.env }} run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6b9ef3c04..311df7052 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ exclude: | ) repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.2 + rev: v0.15.20 hooks: - id: ruff-check args: [ --fix ] @@ -16,7 +16,7 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==25.9.0 + - black==26.5.1 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: @@ -27,6 +27,6 @@ repos: hooks: - id: sphinx-lint - repo: https://github.com/scrapy/sphinx-scrapy - rev: 0.8.6 + rev: 0.8.8 hooks: - id: sphinx-scrapy diff --git a/SECURITY.md b/SECURITY.md index 49cafdf9e..c1a5482c6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,8 +4,8 @@ | Version | Supported | | ------- | ------------------ | -| 2.16.x | :white_check_mark: | -| < 2.16.x | :x: | +| 2.17.x | :white_check_mark: | +| < 2.17.x | :x: | ## Reporting a Vulnerability diff --git a/conftest.py b/conftest.py index cf0568111..b8a9dc19e 100644 --- a/conftest.py +++ b/conftest.py @@ -1,6 +1,6 @@ from __future__ import annotations -import importlib +from importlib.util import find_spec from pathlib import Path from typing import TYPE_CHECKING @@ -11,7 +11,7 @@ from scrapy.utils.reactor import set_asyncio_event_loop_policy from scrapy.utils.reactorless import install_reactor_import_hook from tests.keys import generate_keys from tests.mockserver.http import MockServer -from tests.mockserver.mitm_proxy import MitmProxy +from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd if TYPE_CHECKING: from collections.abc import Generator @@ -50,9 +50,7 @@ if not H2_ENABLED: ) ) -try: - import httpx # noqa: F401 -except ImportError: +if find_spec("httpx2") is None and find_spec("httpx") is None: collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py") @@ -74,40 +72,19 @@ def mockserver() -> Generator[MockServer]: @pytest.fixture # function scope because it modifies os.environ -def mitm_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]: - proxy = MitmProxy() +def proxy_server( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> Generator[str]: + kind = request.param + proxy = MitmProxy(mode="socks5" if kind == "socks5" else None) url = proxy.start() + if kind == "https": + url = url.replace("http://", "https://") monkeypatch.setenv("http_proxy", url) monkeypatch.setenv("https_proxy", url) try: - yield proxy - finally: - proxy.stop() - - -@pytest.fixture # function scope because it modifies os.environ -def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]: - proxy = MitmProxy() - url = proxy.start().replace("http://", "https://") - monkeypatch.setenv("http_proxy", url) - monkeypatch.setenv("https_proxy", url) - - try: - yield proxy - finally: - proxy.stop() - - -@pytest.fixture # function scope because it modifies os.environ -def socks5_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]: - proxy = MitmProxy(mode="socks5") - url = proxy.start() - monkeypatch.setenv("http_proxy", url) - monkeypatch.setenv("https_proxy", url) - - try: - yield proxy + yield kind finally: proxy.stop() @@ -148,15 +125,14 @@ def pytest_runtest_setup(item): "uvloop", "botocore", "boto3", - "mitmproxy", ] for module in optional_deps: - if item.get_closest_marker(f"requires_{module}"): - try: - importlib.import_module(module) - except ImportError: - pytest.skip(f"{module} is not installed") + if item.get_closest_marker(f"requires_{module}") and find_spec(module) is None: + pytest.skip(f"{module} is not installed") + + if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None: + pytest.skip("mitmdump is not available") # Generate localhost certificate files, needed by some tests diff --git a/docs/README.rst b/docs/README.rst deleted file mode 100644 index 4d2236b53..000000000 --- a/docs/README.rst +++ /dev/null @@ -1,68 +0,0 @@ -:orphan: - -====================================== -Scrapy documentation quick start guide -====================================== - -This file provides a quick guide on how to compile the Scrapy documentation. - - -Setup the environment ---------------------- - -To compile the documentation you need Sphinx Python library. To install it -and all its dependencies run the following command from this dir - -:: - - pip install -r requirements.txt - - -Compile the documentation -------------------------- - -To compile the documentation (to classic HTML output) run the following command -from this dir:: - - make html - -Documentation will be generated (in HTML format) inside the ``build/html`` dir. - - -View the documentation ----------------------- - -To view the documentation run the following command:: - - make htmlview - -This command will fire up your default browser and open the main page of your -(previously generated) HTML documentation. - - -Start over ----------- - -To clean up all generated documentation files and start from scratch run:: - - make clean - -Keep in mind that this command won't touch any documentation source files. - - -Recreating documentation on the fly ------------------------------------ - -There is a way to recreate the doc automatically when you make changes, you -need to install watchdog (``pip install watchdog``) and then use:: - - make watch - -Alternative method using tox ----------------------------- - -To compile the documentation to HTML run the following command:: - - tox -e docs - -Documentation will be generated inside the ``docs/_build/all`` dir. diff --git a/docs/conf.py b/docs/conf.py index 99d5df7da..de722baac 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -158,6 +158,8 @@ scrapy_intersphinx_enable = [ "itemloaders", "parsel", "pytest", + "pypug", + "scrapy-lint", "sphinx", "tox", "twisted", diff --git a/docs/contributing.rst b/docs/contributing.rst index c868a0ac4..6d2e08fe8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -323,9 +323,10 @@ deprecation removals are documented in the :ref:`release notes `. Tests ===== -Tests are implemented using the :doc:`Twisted unit-testing framework -`. Running tests requires -:doc:`tox `. +Tests are implemented using pytest_. Running tests requires :doc:`tox +`. + +.. _pytest: https://pytest.org .. _running-tests: @@ -371,6 +372,21 @@ To see coverage report install :doc:`coverage ` see output of ``coverage --help`` for more options like html or xml report. +Some tests need a ``mitmdump`` executable (from mitmproxy_) to test against a +fully featured proxy server; they are skipped when one cannot be found +(``mitmproxy`` is intentionally not a test dependency that would be installed +into test venvs, as that sometimes leads to various dependency conflicts). +To run these tests, make ``mitmdump`` available in one of these ways: + +* install ``mitmproxy`` so that ``mitmdump`` is on your ``PATH``, e.g. with + pipx_ (``pipx install mitmproxy``) or uv_ (``uv tool install mitmproxy``); + +* have uv_ installed, in which case the tests will run + ``uvx --from mitmproxy mitmdump``; + +* set the ``MITMDUMP`` environment variable to the path of a ``mitmdump`` + executable. + Writing tests ------------- @@ -398,3 +414,6 @@ And their unit-tests are in:: .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 .. _test coverage: https://app.codecov.io/gh/scrapy/scrapy +.. _mitmproxy: https://mitmproxy.org/ +.. _pipx: https://pipx.pypa.io/ +.. _uv: https://docs.astral.sh/uv/ diff --git a/docs/faq.rst b/docs/faq.rst index df90122f5..8f2013581 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -285,7 +285,8 @@ consume a lot of memory. In order to avoid parsing all the entire feed at once in memory, you can use the :func:`~scrapy.utils.iterators.xmliter_lxml` and :func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what -:class:`~scrapy.spiders.XMLFeedSpider` uses. +:class:`~scrapy.spiders.XMLFeedSpider` and +:class:`~scrapy.spiders.CSVFeedSpider` use. .. autofunction:: scrapy.utils.iterators.xmliter_lxml @@ -360,10 +361,11 @@ method for this purpose. For example: def process_spider_output(self, response, result): for item_or_request in result: if isinstance(item_or_request, Request): + yield item_or_request continue - adapter = ItemAdapter(item) + adapter = ItemAdapter(item_or_request) for _ in range(adapter["multiply_by"]): - yield deepcopy(item) + yield deepcopy(item_or_request) Does Scrapy support IPv6 addresses? ----------------------------------- @@ -382,8 +384,9 @@ How to deal with ``: filedescriptor out of range in select() ---------------------------------------------------------------------------------------------- This issue `has been reported`_ to appear when running broad crawls in macOS, where the default -Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a -different reactor is possible by using the :setting:`TWISTED_REACTOR` setting. +Twisted reactor was :class:`twisted.internet.selectreactor.SelectReactor` at that time. +If you have switched to this reactor using the :setting:`TWISTED_REACTOR` setting you can switch +to a different one in the same way. .. _faq-stop-response-download: @@ -409,7 +412,6 @@ How can I make a blank request? from scrapy import Request - blank_request = Request("data:,") In this case, the URL is set to a data URI scheme. Data URLs allow you to include data diff --git a/docs/index.rst b/docs/index.rst index a46a2ad9f..688cab81b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ Having trouble? We'd like to help! * Ask or search questions in `StackOverflow using the scrapy tag`_. * Ask or search questions in the `Scrapy subreddit`_. * Search for questions on the archives of the `scrapy-users mailing list`_. -* Ask a question in the `#scrapy IRC channel`_, +* Ask a question in the `#scrapy IRC channel`_. * Report bugs with Scrapy in our `issue tracker`_. * Join the Discord community `Scrapy Discord`_. @@ -91,15 +91,15 @@ Basic concepts :doc:`topics/selectors` Extract the data from web pages using XPath. -:doc:`topics/shell` - Test your extraction code in an interactive environment. - :doc:`topics/items` Define the data you want to scrape. :doc:`topics/loaders` Populate your items with the extracted data. +:doc:`topics/shell` + Test your extraction code in an interactive environment. + :doc:`topics/item-pipeline` Post-process and store your scraped data. @@ -151,6 +151,7 @@ Solving specific problems topics/debug topics/contracts topics/practices + topics/security topics/broad-crawls topics/developer-tools topics/dynamic-content @@ -175,6 +176,10 @@ Solving specific problems :doc:`topics/practices` Get familiar with some Scrapy common practices. +:doc:`topics/security` + Understand the security implications of Scrapy defaults and how to harden + them. + :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 8cef04ff1..cba8c15a1 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -89,6 +89,56 @@ just like any other Python package. (See :ref:`platform-specific guides ` below for non-Python dependencies that you may need to install beforehand). +.. _extras: + +Optional extras +=============== + +Scrapy provides optional :ref:`extras ` +that install additional dependencies to enable specific features. To install +Scrapy with one or more extras, list them in square brackets: + +.. code-block:: console + + pip install scrapy[s3,images] + +The following extras are available: + +.. list-table:: + :header-rows: 1 + + * - Extra + - Provides + * - ``bpython`` + - :ref:`bpython shell ` + * - ``brotli`` + - :ref:`Brotli response decompression ` + * - ``gcs`` + - :ref:`Google Cloud Storage ` for + :ref:`feed exports ` and + :ref:`media pipelines ` + * - ``httpx`` + - :ref:`httpx-handler`, including its HTTP/2 and SOCKS proxy support + * - ``images`` + - :ref:`Images pipeline ` + * - ``ipython`` + - :ref:`IPython shell ` + * - ``ptpython`` + - :ref:`ptpython shell ` + * - ``robotparser`` + - :ref:`Robotexclusionrulesparser robots.txt parsing ` + * - ``s3`` + - :ref:`Amazon S3 ` storage for + :ref:`feed exports `, + :ref:`media pipelines `, and + :ref:`S3 downloads ` + * - ``twisted-http2`` + - :ref:`twisted-http2-handler` + * - ``uvloop`` + - `uvloop `_ event loop + * - ``zstd`` + - :ref:`Zstandard response decompression ` + .. _intro-install-platform-notes: @@ -230,8 +280,8 @@ Installing Scrapy with PyPy on Windows is not tested. You can check that Scrapy is installed correctly by running ``scrapy bench``. If this command gives errors such as ``TypeError: ... got 2 unexpected keyword arguments``, this means -that setuptools was unable to pick up one PyPy-specific dependency. -To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``. +that the ``PyPyDispatcher`` dependency wasn't installed. To fix this issue, run +``pip install 'PyPyDispatcher>=2.1.0'``. .. _intro-install-troubleshooting: diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index ee91ce7ca..5937c5860 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -83,16 +83,17 @@ While this enables you to do very fast crawls (sending multiple concurrent requests at the same time, in a fault-tolerant way) Scrapy also gives you control over the politeness of the crawl through :ref:`a few settings `. You can do things like setting a download delay between -each request, limiting the amount of concurrent requests per domain or per IP, and +each request, limiting the amount of concurrent requests per domain, and even :ref:`using an auto-throttling extension ` that tries to figure these settings out automatically. .. note:: This is using :ref:`feed exports ` to generate the - JSON file, you can easily change the export format (XML or CSV, for example) or the - storage backend (FTP or `Amazon S3`_, for example). You can also write an - :ref:`item pipeline ` to store the items in a database. + JSON Lines file, you can easily change the export format (XML or CSV, for + example) or the storage backend (FTP or `Amazon S3`_, for example). You can + also write an :ref:`item pipeline ` to store the + items in a database. .. _topics-whatelse: diff --git a/docs/news.rst b/docs/news.rst index b8b976df9..8f8477eaa 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,352 @@ Release notes ============= +Scrapy VERSION (unreleased) +--------------------------- + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The following runtime usage of zope.interface_ interfaces is removed: + + - :class:`~scrapy.spiderloader.SpiderLoader` and + :class:`~scrapy.spiderloader.DummySpiderLoader` are no longer marked + as implementing the ``ISpiderLoader`` interface. + + - :func:`~scrapy.spiderloader.get_spider_loader` no longer checks that the + configured spider loader implements the ``ISpiderLoader`` interface. + + - :class:`~scrapy.extensions.feedexport.BlockingFeedStorage`, + :class:`~scrapy.extensions.feedexport.FileFeedStorage` and + :class:`~scrapy.extensions.feedexport.StdoutFeedStorage` are no longer + marked as implementing the ``IFeedStorage`` interface. + + (:issue:`6585`, :issue:`7731`) + +.. _release-2.17.0: + +Scrapy 2.17.0 (2026-07-07) +-------------------------- + +Highlights: + +- Security bug fixes + +- HTTP/2 and SOCKS proxy support for ``HttpxDownloadHandler`` + +- Improved settings for changing allowed TLS versions + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- ``s3://`` requests now use HTTPS by default, instead of plaintext HTTP. + + Previously, :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler` + sent signed S3 requests over plaintext HTTP unless + ``request.meta["is_secure"]`` was set to a true value, exposing the request + path, the AWS ``Authorization`` header, the ``X-Amz-Security-Token`` header + (when using temporary credentials), and the response contents to network + attackers, who could also tamper with responses. See the `76g3-c3x4-crvx`_ + security advisory for details. + + To restore the previous behavior for a given request, set + ``request.meta["is_secure"]`` to ``False``. + + .. _76g3-c3x4-crvx: https://github.com/scrapy/scrapy/security/advisories/GHSA-76g3-c3x4-crvx + +Deprecations +~~~~~~~~~~~~ + +- The ``DOWNLOADER_CLIENT_TLS_METHOD`` setting is deprecated. You should use + the :setting:`DOWNLOAD_TLS_MIN_VERSION` and/or + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings instead if you want to change + the TLS method selection. + (:issue:`3288`, :issue:`6546`) + +- The following spider attributes are deprecated in favor of settings: + + - ``http_user`` (use :setting:`HTTPAUTH_USER`) + + - ``http_pass`` (use :setting:`HTTPAUTH_PASS`) + + - ``http_auth_domain`` (use :setting:`HTTPAUTH_DOMAIN`) + + (:issue:`7590`) + +- The ``scrapy.commands.ScrapyCommand.help()`` method is deprecated. It was + never called by Scrapy. + (:issue:`7626`, :issue:`7633`) + +- The following TLS-related functions and constants, intended for internal + use, are deprecated: + + - ``scrapy.core.downloader.tls.METHOD_TLS`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv10`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv11`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv12`` + + - ``scrapy.core.downloader.tls.openssl_methods`` + + - ``scrapy.core.downloader.tls.DEFAULT_CIPHERS`` + + - ``scrapy.utils.ssl.ffi_buf_to_string()`` + + - ``scrapy.utils.ssl.get_temp_key_info()`` + + - ``scrapy.utils.ssl.x509name_to_string()`` + + (:issue:`6546`, :issue:`7619`, :issue:`7665`) + +- The ``CRAWLSPIDER_FOLLOW_LINKS`` setting is deprecated. You can set + ``follow=False`` in your rules to achieve the same effect. + (:issue:`7592`) + +- Instantiating + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + without a ``crawler`` argument is deprecated. + (:issue:`7655`) + +- Instantiating + :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` without a + ``settings`` argument is deprecated. + (:issue:`7664`) + +New features +~~~~~~~~~~~~ + +- Added support for HTTP/2 requests to + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. It + requires setting the new :setting:`HTTPX_HTTP2_ENABLED` setting to + ``True``. + (:issue:`7575`) + +- Added support for SOCKS proxies to + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. + (:issue:`747`, :issue:`7575`) + +- Added :setting:`DOWNLOAD_TLS_MIN_VERSION` and + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings as replacements for the + ``DOWNLOADER_CLIENT_TLS_METHOD`` setting (which is now deprecated). + Compared to the old setting, they support specifying a range of allowed + versions and support newer TLS versions. + (:issue:`4821`, :issue:`6546`) + +- Added :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS` and + :setting:`HTTPAUTH_DOMAIN` settings and :reqmeta:`http_user`, + :reqmeta:`http_pass` and :reqmeta:`http_auth_domain` meta keys as more + flexible ways to set HTTP authentication data. + (:issue:`7590`) + +- Added a :reqmeta:`verbatim_url` meta key that can be set to ``True`` to + skip request URL canonicalization. + (:issue:`7473`) + +- Added ``deny_tags`` and ``deny_attrs`` arguments to :class:`LinkExtractor + `. + (:issue:`6321`, :issue:`7679`) + +- :attr:`scrapy.Item.fields` now returns the fields in the definition order + instead of the alphabetical one. + (:issue:`7015`, :issue:`7694`) + +- Added a :setting:`RETRY_GIVE_UP_LOG_LEVEL` setting, a + :reqmeta:`give_up_log_level` meta key and a ``give_up_log_level`` argument + of the + :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function that + allow changing the log level of the message logged when the retry limit has + been reached. + (:issue:`4622`, :issue:`5297`, :issue:`7567`) + +- It's now possible to set :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` to + ``None`` to use the default ciphers of the underlying TLS implementation. + (:issue:`7499`, :issue:`7665`) + +Improvements +~~~~~~~~~~~~ + +- :class:`~scrapy.FormRequest` is no longer deprecated, only its + ``from_response()`` method is still deprecated. + (:issue:`7561`, :issue:`7671`) + +- Switched the item definition in the default project template from a + :class:`scrapy.item.Item` to a dataclass. + (:issue:`7493`, :issue:`7513`) + +- Fixed deprecation warnings with pyOpenSSL 26.3.0. + (:issue:`7619`) + +- Removed the runtime warnings for :attr:`Spider.allowed_domains + ` containing URLs or domains with ports + instead of just domains and for spider classes having a ``start_url`` + attribute instead of :class:`~scrapy.spiders.Spider.start_urls`. Please use + :doc:`scrapy-lint ` to find mistakes in your spider code + instead. + (:issue:`4421`, :issue:`7627`) + +- :func:`scrapy.utils.test.get_crawler` now disables + :setting:`TELNETCONSOLE_ENABLED` by default. + (:issue:`7644`) + +- Other code refactoring and improvements. + (:issue:`7409`, :issue:`7593`, :issue:`7594`, :issue:`7611`, :issue:`7649`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` no + longer ignores proxy credentials for redirected or retried requests. + (:issue:`7601`, :issue:`7630`) + +- :class:`~scrapy.extensions.feedexport.GCSFeedStorage` now closes the + temporary file after the upload. + (:issue:`7546`) + +- Fixed ``scrapy shell `` running a full spider crawl when there is a + spider for the requested URL. This bug was introduced in Scrapy 2.13.0. + (:issue:`7552`, :issue:`7557`) + +- The :setting:`IMAGES_STORE_S3_ACL` and :setting:`IMAGES_STORE_GCS_ACL` + settings are no longer ignored. This bug was introduced in Scrapy 2.12.0. + (:issue:`7597`, :issue:`7614`) + +- :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` now closes + the connection after making the request. + (:issue:`7602`, :issue:`7667`) + +- Removed the deprecated ``spider`` argument from the pipeline defined in the + default project template. + (:issue:`7676`) + +- Fixed ``scrapy genspider --edit`` not working. + (:issue:`7260`, :issue:`7683`) + +- When a :class:`~scrapy.crawler.Crawler` instance is passed to + :meth:`AsyncCrawlerRunner.create_crawler() + ` or + :meth:`CrawlerRunner.create_crawler() + `, settings from both classes + are now merged, previously only the settings from the + :class:`~scrapy.crawler.Crawler` instance were used. + (:issue:`1280`, :issue:`7647`) + +- Fixed several issues with cookie handling in + :func:`scrapy.utils.request.request_to_curl`. + (:issue:`7603`, :issue:`7675`, :issue:`7684`) + +- Fixed :class:`scrapy.resolver.CachingThreadedResolver` not disabling the + cache when :setting:`DNSCACHE_ENABLED` is set to ``False``. + (:issue:`7663`) + +- Fixed :func:`scrapy.utils.response.open_in_browser` not removing comments + when looking for the ```` tag. + (:issue:`7506`) + +- Fixed checking for deprecated methods in custom :setting:`ITEM_PROCESSOR` + implementations. + (:issue:`7589`) + +- Fixed :func:`scrapy.utils.url.strip_url` corrupting some URLs with + credentials. + (:issue:`7604`, :issue:`7605`) + +- :func:`scrapy.utils.misc.rel_has_nofollow` now ignores the case when + looking for "nofollow" strings. + (:issue:`7632`) + +- Fixed an exception in :class:`scrapy.utils.sitemap.Sitemap` when parsing + some malformed sitemaps. + (:issue:`7686`, :issue:`7687`) + +Documentation +~~~~~~~~~~~~~ + +- Mentioned :doc:`scrapy-lint ` in the docs. + (:issue:`4421`, :issue:`7627`) + +- Added the docs about :ref:`security considerations `. + (:issue:`7389`, :issue:`7678`) + +- Improved the :ref:`item pipeline docs `. + (:issue:`2350`, :issue:`7676`) + +- Documented which stats are collected by + :class:`~scrapy.extensions.corestats.CoreStats`. + (:issue:`7421`) + +- Switched documentation examples from using :class:`scrapy.item.Item` to + using dataclasses. + (:issue:`7493`, :issue:`7513`) + +- Added feature comparison tables to the :ref:`download handler + ` docs. + (:issue:`7575`) + +- Improved the docs for :ref:`logging settings `. + (:issue:`6909`, :issue:`7668`) + +- Documented a way to :ref:`improve startup time and memory usage + ` by using :setting:`SPIDER_MODULES`. + (:issue:`7576`, :issue:`7600`) + +- Clarified handling of the ``type`` argument of :class:`~scrapy.Selector`. + (:issue:`7704`) + +- Other documentation improvements and fixes. + (:issue:`4954`, + :issue:`6120`, + :issue:`7286`, + :issue:`7564`, + :issue:`7573`, + :issue:`7598`, + :issue:`7599`, + :issue:`7698`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Fixed deprecation warnings with pytest 9.1.0. + (:issue:`7621`) + +- Type hints improvements and fixes. + (:issue:`6958`, :issue:`7586`) + +- CI and test improvements and fixes. + (:issue:`5954`, + :issue:`7002`, + :issue:`7017`, + :issue:`7247`, + :issue:`7508`, + :issue:`7545`, + :issue:`7566`, + :issue:`7574`, + :issue:`7585`, + :issue:`7595`, + :issue:`7608`, + :issue:`7610`, + :issue:`7612`, + :issue:`7616`, + :issue:`7625`, + :issue:`7637`, + :issue:`7639`, + :issue:`7640`, + :issue:`7641`, + :issue:`7642`, + :issue:`7643`, + :issue:`7644`, + :issue:`7645`, + :issue:`7646`, + :issue:`7654`, + :issue:`7655`, + :issue:`7664`, + :issue:`7672`, + :issue:`7677`, + :issue:`7680`, + :issue:`7682`, + :issue:`7692`) + .. _release-2.16.0: Scrapy 2.16.0 (2026-05-19) @@ -711,7 +1057,7 @@ Deprecations New features ~~~~~~~~~~~~ -- Added a new setting, :setting:`REFERER_POLICIES`, to allow customizing +- Added a new setting, :setting:`REFERRER_POLICIES`, to allow customizing supported referrer policies. Bug fixes @@ -2319,7 +2665,7 @@ Deprecation removals (:issue:`6109`, :issue:`6116`) - A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that - does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface + does not implement the ``ISpiderLoader`` interface will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at run time. Non-compliant classes have been triggering a deprecation warning since Scrapy 1.0.0. @@ -4809,7 +5155,7 @@ Bug fixes * The system file mode creation mask no longer affects the permissions of files generated using the :command:`startproject` command (:issue:`4722`) -* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names +* ``scrapy.utils.iterators.xmliter`` now supports namespaced node names (:issue:`861`, :issue:`4746`) * :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can @@ -5722,7 +6068,7 @@ The following changes may impact custom priority queue classes: * A new keyword parameter has been added: ``key``. It is a string that is always an empty string for memory queues and indicates the - :setting:`JOB_DIR` value for disk queues. + :setting:`JOBDIR` value for disk queues. * The parameter for disk queues that contains data from the previous crawl, ``startprios`` or ``slot_startprios``, is now passed as a @@ -6243,8 +6589,8 @@ Backward-incompatible changes ``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly. * :class:`~scrapy.crawler.Crawler`, - :class:`CrawlerRunner.crawl ` and - :class:`CrawlerRunner.create_crawler ` + :meth:`CrawlerRunner.crawl ` and + :meth:`CrawlerRunner.create_crawler ` no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they only accept a :class:`~scrapy.spiders.Spider` subclass now. @@ -6276,7 +6622,7 @@ New features ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be :ref:`enabled ` for a significant scheduling improvement on crawls targeting multiple web domains, at the - cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) + cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`) * A new :attr:`.Request.cb_kwargs` attribute provides a cleaner way to pass keyword arguments to callback methods @@ -8755,7 +9101,7 @@ New features and settings - In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`) - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: - - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP` + - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP`` - check the documentation for more details - Added builtin caching DNS resolver (:rev:`2728`) - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) diff --git a/docs/requirements.in b/docs/requirements.in index 140791641..a1f3a7468 100644 --- a/docs/requirements.in +++ b/docs/requirements.in @@ -5,4 +5,4 @@ sphinx sphinx-notfound-page sphinx-rtd-theme sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 diff --git a/docs/requirements.txt b/docs/requirements.txt index 9c93dacd0..a5cbad302 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0 # via # -r docs/requirements.in # sphinx-rtd-dark-mode -sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@b1d55db4d16a5425fc68576d63519bbfe26dd9c0 +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737 # via -r docs/requirements.in sphinx-sitemap==2.9.0 # via sphinx-scrapy diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 01e4bcac1..75f15b3ae 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -21,10 +21,14 @@ The ``ADDONS`` setting is a dict in which every key is an add-on class or its import path and the value is its priority. This is an example where two add-ons are enabled in a project's -``settings.py``:: +``settings.py``: + +.. skip: next + +.. code-block:: python ADDONS = { - 'path.to.someaddon': 0, + "path.to.someaddon": 0, SomeAddonClass: 1, } @@ -56,7 +60,9 @@ the following methods: :type settings: :class:`~scrapy.settings.BaseSettings` The settings set by the add-on should use the ``addon`` priority (see -:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: +:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`): + +.. code-block:: python class MyAddon: def update_settings(self, settings): @@ -168,7 +174,6 @@ Use a fallback component: from scrapy.utils.misc import build_from_crawler, load_object - FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" diff --git a/docs/topics/api.rst b/docs/topics/api.rst index 19082d9d7..7ff8d3464 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -172,46 +172,15 @@ SpiderLoader API .. module:: scrapy.spiderloader :synopsis: The spider loader -.. class:: SpiderLoader +Custom spider loaders can be employed by specifying their path in the +:setting:`SPIDER_LOADER_CLASS` project setting. They must implement +:class:`SpiderLoaderProtocol`. - This class is in charge of retrieving and handling the spider classes - defined across the project. +.. autoclass:: SpiderLoaderProtocol + :members: - Custom spider loaders can be employed by specifying their path in the - :setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement - the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an - errorless execution. - - .. method:: from_settings(settings) - - This class method is used by Scrapy to create an instance of the class. - It's called with the current project settings, and it loads the spiders - found recursively in the modules of the :setting:`SPIDER_MODULES` - setting. - - :param settings: project settings - :type settings: :class:`~scrapy.settings.Settings` instance - - .. method:: load(spider_name) - - Get the Spider class with the given name. It'll look into the previously - loaded spiders for a spider class with name ``spider_name`` and will raise - a KeyError if not found. - - :param spider_name: spider class name - :type spider_name: str - - .. method:: list() - - Get the names of the available spiders in the project. - - .. method:: find_by_request(request) - - List the spiders' names that can handle the given request. Will try to - match the request's url against the domains of the spiders. - - :param request: queried request - :type request: :class:`~scrapy.Request` instance +.. autoclass:: SpiderLoader + :members: .. autoclass:: DummySpiderLoader diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 1a151044f..63c217e93 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -5,7 +5,7 @@ asyncio ======= Scrapy supports :mod:`asyncio` natively. New projects created with -:command:`scrapy startproject` have asyncio enabled by default, and you can use +:command:`startproject` have asyncio enabled by default, and you can use :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine `. @@ -18,7 +18,7 @@ no additional setup is needed. Configuring the asyncio reactor =============================== -New projects generated with :command:`scrapy startproject` have the asyncio +New projects generated with :command:`startproject` have the asyncio reactor configured by default. No manual setup is needed. The :setting:`TWISTED_REACTOR` setting controls which Twisted reactor Scrapy @@ -105,6 +105,9 @@ Scrapy API requires passing a Deferred to it) using the following helpers: .. autofunction:: scrapy.utils.defer.deferred_from_coro .. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f + +The following function helps with a reverse wrapping: + .. autofunction:: scrapy.utils.defer.ensure_awaitable @@ -147,6 +150,12 @@ Using Scrapy without a Twisted reactor .. warning:: This is currently experimental and may not be suitable for production use. +.. note:: As the Twisted download handlers cannot be used without a reactor, + the default download handler in this mode is + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. You + will need to additionally install the :ref:`httpx ` extra to use + it, unless you switch to some different handler. + It's possible to use Scrapy without installing a Twisted reactor at all, by setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this mode Scrapy will use the asyncio event loop directly, and most of the Scrapy @@ -190,7 +199,7 @@ in future Scrapy versions. The following features are not available: :class:`~scrapy.crawler.CrawlerProcess` (:class:`~scrapy.crawler.AsyncCrawlerProcess` and :class:`~scrapy.crawler.AsyncCrawlerRunner` are available) -* Twisted-specific DNS resolvers (the :setting:`DNS_RESOLVER` setting) +* Twisted-specific DNS resolvers (the :setting:`TWISTED_DNS_RESOLVER` setting) * User and 3rd-party code that requires a reactor (see :ref:`below ` for examples) @@ -218,7 +227,8 @@ for its differences and limitations compared to Additionally, :class:`~scrapy.crawler.AsyncCrawlerProcess` will install a :term:`meta path finder` that prevents :mod:`twisted.internet.reactor` from -being imported. +being imported. It will be uninstalled when :meth:`AsyncCrawlerProcess.start() +` exits. .. _asyncio-without-reactor-migrate: @@ -315,8 +325,7 @@ implementations, :class:`~asyncio.ProactorEventLoop` (default) and :class:`~asyncio.SelectorEventLoop` works with Twisted. Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop` -automatically when you change the :setting:`TWISTED_REACTOR` setting or call -:func:`~scrapy.utils.reactor.install_reactor`. +automatically when installing the asyncio reactor. .. note:: Other libraries you use may require :class:`~asyncio.ProactorEventLoop`, e.g. because it supports diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index d0321c906..4f28019da 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -75,7 +75,7 @@ AutoThrottle algorithm adjusts download delays based on the following rules: .. _download-latency: In Scrapy, the download latency is measured as the time elapsed between -establishing the TCP connection and receiving the HTTP headers. +sending the request and receiving the HTTP headers. Note that these latencies are very hard to measure accurately in a cooperative multitasking environment because Scrapy may be busy processing a spider @@ -88,6 +88,8 @@ server) is, and this extension builds on that premise. Prevent specific requests from triggering slot delay adjustments ================================================================ +.. versionadded:: 2.12.0 + AutoThrottle adjusts the delay of download slots based on the latencies of responses that belong to that download slot. The only exceptions are non-200 responses, which are only taken into account to increase that delay, but diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8d1351eb9..ee2c3a3cd 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -199,6 +199,7 @@ Global commands: * :command:`fetch` * :command:`view` * :command:`version` +* :command:`bench` Project-only commands: @@ -207,7 +208,6 @@ Project-only commands: * :command:`list` * :command:`edit` * :command:`parse` -* :command:`bench` .. command:: startproject @@ -309,11 +309,25 @@ Usage examples:: * parse_item $ scrapy check - [FAILED] first_spider:parse_item - >>> 'RetailPricex' field is missing + F.F. + ====================================================================== + FAIL: [first_spider] parse (@returns post-hook) + ---------------------------------------------------------------------- + Traceback (most recent call last): + ... + scrapy.exceptions.ContractFail: Returned 92 requests, expected 0..4 - [FAILED] first_spider:parse - >>> Returned 92 requests, expected 0..4 + ====================================================================== + FAIL: [first_spider] parse_item (@scrapes post-hook) + ---------------------------------------------------------------------- + Traceback (most recent call last): + ... + scrapy.exceptions.ContractFail: Missing fields: RetailPricex + + ---------------------------------------------------------------------- + Ran 4 contracts in 0.174s + + FAILED (failures=2) .. skip: end @@ -377,7 +391,7 @@ Supported options: * ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider -* ``--headers``: print the response's HTTP headers instead of the response's body +* ``--headers``: print the request's and response's HTTP headers instead of the response's body * ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) @@ -387,15 +401,19 @@ Usage examples:: [ ... html content here ... ] $ scrapy fetch --nolog --headers http://www.example.com/ - {'Accept-Ranges': ['bytes'], - 'Age': ['1263 '], - 'Connection': ['close '], - 'Content-Length': ['596'], - 'Content-Type': ['text/html; charset=UTF-8'], - 'Date': ['Wed, 18 Aug 2010 23:59:46 GMT'], - 'Etag': ['"573c1-254-48c9c87349680"'], - 'Last-Modified': ['Fri, 30 Jul 2010 15:30:18 GMT'], - 'Server': ['Apache/2.2.3 (CentOS)']} + > Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + > Accept-Language: en + > User-Agent: Scrapy/2.16.0 (+https://scrapy.org) + > Accept-Encoding: gzip, deflate, br + > + < Date: Wed, 08 Jul 2026 06:15:01 GMT + < Content-Type: text/html + < Server: cloudflare + < Last-Modified: Wed, 01 Jul 2026 17:50:18 GMT + < Allow: GET, HEAD + < Cf-Cache-Status: HIT + < Age: 8184 + < Cf-Ray: a17cf3b80eddf141-DME .. command:: view @@ -476,7 +494,7 @@ Supported options: * ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider -* ``--a NAME=VALUE``: set spider argument (may be repeated) +* ``-a NAME=VALUE``: set spider argument (may be repeated) * ``--callback`` or ``-c``: spider method to use as callback for parsing the response @@ -605,7 +623,10 @@ shouldn't matter to the user running the command, but when the user :ref:`needs a non-default Twisted reactor `, it may be important. Scrapy decides which of these two classes to use based on the value of the -:setting:`TWISTED_REACTOR` setting. If the setting value is the default one +:setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED` settings. +With :setting:`TWISTED_REACTOR_ENABLED` set to ``False`` it will use +:class:`~scrapy.crawler.AsyncCrawlerProcess`. Otherwise, if the +:setting:`TWISTED_REACTOR` value is the default one (``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``), :class:`~scrapy.crawler.AsyncCrawlerProcess` will be used, otherwise :class:`~scrapy.crawler.CrawlerProcess` will be used. The :ref:`spider settings diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 116aac323..9dcd9d69c 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -16,7 +16,7 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): -- The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be +- The :meth:`~scrapy.Spider.start` spider method, which *must* be defined as an :term:`asynchronous generator`. .. versionadded:: 2.13 @@ -204,13 +204,15 @@ This means you can use many useful Python libraries providing such code: Common use cases for asynchronous code include: * requesting data from websites, databases and other services (in - :meth:`~scrapy.spiders.Spider.start`, callbacks, pipelines and + :meth:`~scrapy.Spider.start`, callbacks, pipelines and middlewares); * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the :signal:`spider_opened` handler); -* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download` - (see :ref:`the screenshot pipeline example`). +* calling asynchronous Scrapy methods like + :meth:`ExecutionEngine.download_async() + ` (see :ref:`the + screenshot pipeline example `). .. _aio-libs: https://github.com/aio-libs @@ -268,5 +270,5 @@ You can also send multiple requests in parallel: yield { "h1": response.css("h1::text").get(), "price": responses[0].css(".price::text").get(), - "price2": responses[1].css(".color::text").get(), + "color": responses[1].css(".color::text").get(), } diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 05dffcdda..a5ffe00f1 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -246,7 +246,6 @@ also request each page to get every quote on the site: .. code-block:: python import scrapy - import json class QuoteSpider(scrapy.Spider): @@ -256,7 +255,7 @@ also request each page to get every quote on the site: start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"] def parse(self, response): - data = json.loads(response.text) + data = response.json() for quote in data["quotes"]: yield {"quote": quote["text"]} if data["has_next"]: diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 888bfaf08..34ab4f105 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -39,6 +39,10 @@ for additional schemes and to replace or disable default ones: "sftp": "my.download_handlers.SftpHandler", } +.. seealso:: :ref:`security-unencrypted-protocols` and + :ref:`security-local-resources`, for the security implications of the + default ``http``, ``ftp``, ``file`` and ``data`` handlers. + Replacing HTTP(S) download handlers ----------------------------------- @@ -85,7 +89,7 @@ the following API: If ``True``, the handler will only be instantiated when the first request handled by it needs to be downloaded. - .. method:: download_request(request: Request) -> Response: + .. method:: download_request(request: Request) -> Response :async: Download the given request and return a response. @@ -169,6 +173,8 @@ of this package for more information. H2DownloadHandler ----------------- +.. note:: Requires the :ref:`twisted-http2 ` extra. + .. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler | Supported scheme: ``https``. @@ -181,9 +187,6 @@ for them. It's implemented using :mod:`twisted.web.client` and the ``h2`` library. -For this handler to work you need to install the ``Twisted[http2]`` extra -dependency. - If you want to use this handler you need to replace the default one for the ``https`` scheme: @@ -269,9 +272,13 @@ Other limitations: - HTTPS proxies to HTTPS destinations are not supported. +.. _httpx-handler: + HttpxDownloadHandler -------------------- +.. note:: Requires the :ref:`httpx ` extra. + .. versionadded:: 2.15.0 .. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler @@ -284,7 +291,9 @@ HttpxDownloadHandler This handler supports ``http://host/path`` and ``https://host/path`` URLs and uses the HTTP/1.1 or HTTP/2 protocol for them. -It's implemented using the ``httpx`` library and needs it to be installed. +It's implemented using the httpx2_ library. + +.. _httpx2: https://httpx2.pydantic.dev/ If you want to use this handler you need to replace the default ones for the ``http`` and ``https`` schemes: @@ -307,8 +316,8 @@ Features and limitations =========================== ======================================= HTTP proxies Yes -SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``) -HTTP/2 Yes (requires ``httpx[http2]``) +SOCKS proxies Yes (SOCKS5) +HTTP/2 Yes ``response.certificate`` DER bytes Per-request ``bindaddress`` No (not supported by the library) TLS implementation Standard library ``ssl`` @@ -325,12 +334,11 @@ Other limitations: HTTPX_HTTP2_ENABLED ^^^^^^^^^^^^^^^^^^^ +.. versionadded:: 2.17.0 + Default: ``False`` -Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra -needs to be installed if you want to enable this setting. - -.. versionadded:: VERSION +Whether to enable HTTP/2 support in this handler. Built-in non-HTTP download handlers reference ============================================= @@ -374,9 +382,13 @@ This handler supports ``ftp://host/path`` FTP URIs. It's implemented using :mod:`twisted.protocols.ftp`. +.. _s3-handler: + S3DownloadHandler ----------------- +.. note:: Requires the :ref:`s3 ` extra. + .. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler | Supported scheme: ``s3``. @@ -386,4 +398,6 @@ S3DownloadHandler This handler supports ``s3://bucket/path`` S3 URIs. -It's implemented using the ``botocore`` library and needs it to be installed. +It's implemented using the botocore_ library. + +.. _botocore: https://github.com/boto/botocore diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0c1af5276..934eb19ac 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -351,6 +351,8 @@ HttpAuthMiddleware HTTPAUTH_USER ~~~~~~~~~~~~~ +.. versionadded:: 2.17.0 + Default: ``""`` The username to use for HTTP basic authentication, applied to all requests @@ -361,6 +363,8 @@ whose URL matches :setting:`HTTPAUTH_DOMAIN`. HTTPAUTH_PASS ~~~~~~~~~~~~~ +.. versionadded:: 2.17.0 + Default: ``""`` The password to use for HTTP basic authentication. @@ -370,6 +374,8 @@ The password to use for HTTP basic authentication. HTTPAUTH_DOMAIN ~~~~~~~~~~~~~~~ +.. versionadded:: 2.17.0 + Default: ``None`` The domain (and its subdomains) to which HTTP basic authentication credentials @@ -379,6 +385,8 @@ that this risks leaking credentials to unrelated domains. This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER` or :setting:`HTTPAUTH_PASS` is set. +.. seealso:: :ref:`security-credential-leakage` + .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication @@ -497,7 +505,7 @@ Filesystem storage backend (default) * ``response_body`` - the plain response body - * ``response_headers`` - the request headers (in raw HTTP format) + * ``response_headers`` - the response headers (in raw HTTP format) * ``meta`` - some metadata of this cache resource in Python ``repr()`` format (grep-friendly format) @@ -539,7 +547,7 @@ defines the methods described below. .. method:: open_spider(spider) This method gets called after a spider has been opened for crawling. It handles - the :signal:`open_spider ` signal. + the :signal:`spider_opened` signal. :param spider: the spider which has been opened :type spider: :class:`~scrapy.Spider` object @@ -547,7 +555,7 @@ defines the methods described below. .. method:: close_spider(spider) This method gets called after a spider has been closed. It handles - the :signal:`close_spider ` signal. + the :signal:`spider_closed` signal. :param spider: the spider which has been closed :type spider: :class:`~scrapy.Spider` object @@ -583,8 +591,8 @@ In order to use your storage backend, set: HTTPCache middleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The :class:`HttpCacheMiddleware` can be configured through the following -settings: +:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` can be +configured through the following settings: .. setting:: HTTPCACHE_ENABLED @@ -719,6 +727,8 @@ We assume that the spider will not issue Cache-Control directives in requests unless it actually needs them, so directives in requests are not filtered. +.. _http-compression: + HttpCompressionMiddleware ------------------------- @@ -730,14 +740,12 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. - This middleware also supports decoding `brotli-compressed`_ as well as - `zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is - installed, respectively. + This middleware also supports decoding `brotli-compressed`_ responses with + the :ref:`brotli ` extra, and `zstd-compressed`_ + responses with the :ref:`zstd ` extra. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt -.. _brotli: https://pypi.org/project/Brotli/ .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt -.. _zstandard: https://pypi.org/project/zstandard/ HttpCompressionMiddleware Settings @@ -806,7 +814,6 @@ HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. setting:: HTTPPROXY_ENABLED -.. setting:: HTTPPROXY_AUTH_ENCODING HTTPPROXY_ENABLED ^^^^^^^^^^^^^^^^^ @@ -815,6 +822,8 @@ Default: ``True`` Whether or not to enable the :class:`HttpProxyMiddleware`. +.. setting:: HTTPPROXY_AUTH_ENCODING + HTTPPROXY_AUTH_ENCODING ^^^^^^^^^^^^^^^^^^^^^^^ @@ -859,9 +868,9 @@ OffsiteMiddleware .. reqmeta:: allow_offsite If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to - ``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then - the OffsiteMiddleware will allow the request even if its domain is not listed - in allowed domains. + ``True`` or :attr:`Request.meta ` has ``allow_offsite`` + set to ``True``, then the OffsiteMiddleware will allow the request even if + its domain is not listed in allowed domains. RedirectMiddleware ------------------ @@ -976,7 +985,7 @@ Whether the Meta Refresh middleware will be enabled. METAREFRESH_IGNORE_TAGS ^^^^^^^^^^^^^^^^^^^^^^^ -Default: ``[]`` +Default: ``["noscript"]`` Meta tags within these tags are ignored. @@ -1006,17 +1015,6 @@ RetryMiddleware A middleware to retry failed requests that are potentially caused by temporary problems such as a connection timeout or HTTP 500 error. -Failed pages are collected on the scraping process and rescheduled at the -end, once the spider has finished crawling all regular (non failed) pages. - -The :class:`RetryMiddleware` can be configured through the following -settings (see the settings documentation for more info): - -* :setting:`RETRY_ENABLED` -* :setting:`RETRY_TIMES` -* :setting:`RETRY_HTTP_CODES` -* :setting:`RETRY_EXCEPTIONS` - .. reqmeta:: dont_retry If :attr:`Request.meta ` has ``dont_retry`` key @@ -1083,7 +1081,7 @@ Default:: 'twisted.internet.error.ConnectionDone', 'twisted.internet.error.ConnectError', 'twisted.internet.error.ConnectionLost', - IOError, + OSError, 'scrapy.core.downloader.handlers.http11.TunnelError', ] @@ -1102,6 +1100,8 @@ exception propagation, see RETRY_GIVE_UP_LOG_LEVEL ^^^^^^^^^^^^^^^^^^^^^^^ +.. versionadded:: 2.17.0 + Default: ``"ERROR"`` :ref:`Logging level ` used for the message logged when a request @@ -1239,8 +1239,7 @@ Based on `Robotexclusionrulesparser ` extra. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.RerpRobotParser`` diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 5a399e094..30b6536c7 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -133,7 +133,7 @@ data from it depends on the type of response: .. code-block:: python - selector = Selector(data["html"]) + selector = Selector(text=data["html"]) - If the response is JavaScript, or HTML with a ``