diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index f51dd9799..15f25d9b8 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -79,9 +79,6 @@ jobs: - python-version: "3.14" env: TOXENV: botocore - - python-version: "3.14" - env: - TOXENV: mitmproxy steps: - uses: actions/checkout@v6 @@ -97,6 +94,9 @@ jobs: sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev + - name: Install mitmproxy + run: pipx install mitmproxy + - name: Run tests env: ${{ matrix.env }} run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 754bd6f9c..311df7052 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ exclude: | ) repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.2 + rev: v0.15.20 hooks: - id: ruff-check args: [ --fix ] @@ -16,7 +16,7 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==25.9.0 + - black==26.5.1 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: diff --git a/SECURITY.md b/SECURITY.md index 49cafdf9e..c1a5482c6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,8 +4,8 @@ | Version | Supported | | ------- | ------------------ | -| 2.16.x | :white_check_mark: | -| < 2.16.x | :x: | +| 2.17.x | :white_check_mark: | +| < 2.17.x | :x: | ## Reporting a Vulnerability diff --git a/conftest.py b/conftest.py index 532f83f56..7403b07b2 100644 --- a/conftest.py +++ b/conftest.py @@ -11,7 +11,7 @@ from scrapy.utils.reactor import set_asyncio_event_loop_policy from scrapy.utils.reactorless import install_reactor_import_hook from tests.keys import generate_keys from tests.mockserver.http import MockServer -from tests.mockserver.mitm_proxy import MitmProxy +from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd if TYPE_CHECKING: from collections.abc import Generator @@ -127,7 +127,6 @@ def pytest_runtest_setup(item): "uvloop", "botocore", "boto3", - "mitmproxy", ] for module in optional_deps: @@ -137,6 +136,9 @@ def pytest_runtest_setup(item): except ImportError: pytest.skip(f"{module} is not installed") + if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None: + pytest.skip("mitmdump is not available") + # Generate localhost certificate files, needed by some tests generate_keys() diff --git a/docs/README.rst b/docs/README.rst deleted file mode 100644 index 4d2236b53..000000000 --- a/docs/README.rst +++ /dev/null @@ -1,68 +0,0 @@ -:orphan: - -====================================== -Scrapy documentation quick start guide -====================================== - -This file provides a quick guide on how to compile the Scrapy documentation. - - -Setup the environment ---------------------- - -To compile the documentation you need Sphinx Python library. To install it -and all its dependencies run the following command from this dir - -:: - - pip install -r requirements.txt - - -Compile the documentation -------------------------- - -To compile the documentation (to classic HTML output) run the following command -from this dir:: - - make html - -Documentation will be generated (in HTML format) inside the ``build/html`` dir. - - -View the documentation ----------------------- - -To view the documentation run the following command:: - - make htmlview - -This command will fire up your default browser and open the main page of your -(previously generated) HTML documentation. - - -Start over ----------- - -To clean up all generated documentation files and start from scratch run:: - - make clean - -Keep in mind that this command won't touch any documentation source files. - - -Recreating documentation on the fly ------------------------------------ - -There is a way to recreate the doc automatically when you make changes, you -need to install watchdog (``pip install watchdog``) and then use:: - - make watch - -Alternative method using tox ----------------------------- - -To compile the documentation to HTML run the following command:: - - tox -e docs - -Documentation will be generated inside the ``docs/_build/all`` dir. diff --git a/docs/contributing.rst b/docs/contributing.rst index c868a0ac4..6d2e08fe8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -323,9 +323,10 @@ deprecation removals are documented in the :ref:`release notes `. Tests ===== -Tests are implemented using the :doc:`Twisted unit-testing framework -`. Running tests requires -:doc:`tox `. +Tests are implemented using pytest_. Running tests requires :doc:`tox +`. + +.. _pytest: https://pytest.org .. _running-tests: @@ -371,6 +372,21 @@ To see coverage report install :doc:`coverage ` see output of ``coverage --help`` for more options like html or xml report. +Some tests need a ``mitmdump`` executable (from mitmproxy_) to test against a +fully featured proxy server; they are skipped when one cannot be found +(``mitmproxy`` is intentionally not a test dependency that would be installed +into test venvs, as that sometimes leads to various dependency conflicts). +To run these tests, make ``mitmdump`` available in one of these ways: + +* install ``mitmproxy`` so that ``mitmdump`` is on your ``PATH``, e.g. with + pipx_ (``pipx install mitmproxy``) or uv_ (``uv tool install mitmproxy``); + +* have uv_ installed, in which case the tests will run + ``uvx --from mitmproxy mitmdump``; + +* set the ``MITMDUMP`` environment variable to the path of a ``mitmdump`` + executable. + Writing tests ------------- @@ -398,3 +414,6 @@ And their unit-tests are in:: .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 .. _test coverage: https://app.codecov.io/gh/scrapy/scrapy +.. _mitmproxy: https://mitmproxy.org/ +.. _pipx: https://pipx.pypa.io/ +.. _uv: https://docs.astral.sh/uv/ diff --git a/docs/faq.rst b/docs/faq.rst index df90122f5..a396a8212 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -285,7 +285,8 @@ consume a lot of memory. In order to avoid parsing all the entire feed at once in memory, you can use the :func:`~scrapy.utils.iterators.xmliter_lxml` and :func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what -:class:`~scrapy.spiders.XMLFeedSpider` uses. +:class:`~scrapy.spiders.XMLFeedSpider` and +:class:`~scrapy.spiders.CSVFeedSpider` use. .. autofunction:: scrapy.utils.iterators.xmliter_lxml @@ -360,10 +361,11 @@ method for this purpose. For example: def process_spider_output(self, response, result): for item_or_request in result: if isinstance(item_or_request, Request): + yield item_or_request continue - adapter = ItemAdapter(item) + adapter = ItemAdapter(item_or_request) for _ in range(adapter["multiply_by"]): - yield deepcopy(item) + yield deepcopy(item_or_request) Does Scrapy support IPv6 addresses? ----------------------------------- @@ -409,7 +411,6 @@ How can I make a blank request? from scrapy import Request - blank_request = Request("data:,") In this case, the URL is set to a data URI scheme. Data URLs allow you to include data diff --git a/docs/index.rst b/docs/index.rst index a46a2ad9f..688cab81b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ Having trouble? We'd like to help! * Ask or search questions in `StackOverflow using the scrapy tag`_. * Ask or search questions in the `Scrapy subreddit`_. * Search for questions on the archives of the `scrapy-users mailing list`_. -* Ask a question in the `#scrapy IRC channel`_, +* Ask a question in the `#scrapy IRC channel`_. * Report bugs with Scrapy in our `issue tracker`_. * Join the Discord community `Scrapy Discord`_. @@ -91,15 +91,15 @@ Basic concepts :doc:`topics/selectors` Extract the data from web pages using XPath. -:doc:`topics/shell` - Test your extraction code in an interactive environment. - :doc:`topics/items` Define the data you want to scrape. :doc:`topics/loaders` Populate your items with the extracted data. +:doc:`topics/shell` + Test your extraction code in an interactive environment. + :doc:`topics/item-pipeline` Post-process and store your scraped data. @@ -151,6 +151,7 @@ Solving specific problems topics/debug topics/contracts topics/practices + topics/security topics/broad-crawls topics/developer-tools topics/dynamic-content @@ -175,6 +176,10 @@ Solving specific problems :doc:`topics/practices` Get familiar with some Scrapy common practices. +:doc:`topics/security` + Understand the security implications of Scrapy defaults and how to harden + them. + :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index ee91ce7ca..488e1c2e3 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -90,9 +90,10 @@ to figure these settings out automatically. .. note:: This is using :ref:`feed exports ` to generate the - JSON file, you can easily change the export format (XML or CSV, for example) or the - storage backend (FTP or `Amazon S3`_, for example). You can also write an - :ref:`item pipeline ` to store the items in a database. + JSON Lines file, you can easily change the export format (XML or CSV, for + example) or the storage backend (FTP or `Amazon S3`_, for example). You can + also write an :ref:`item pipeline ` to store the + items in a database. .. _topics-whatelse: diff --git a/docs/news.rst b/docs/news.rst index b8b976df9..63f0d69e4 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,330 @@ Release notes ============= +.. _release-2.17.0: + +Scrapy 2.17.0 (2026-07-07) +-------------------------- + +Highlights: + +- Security bug fixes + +- HTTP/2 and SOCKS proxy support for ``HttpxDownloadHandler`` + +- Improved settings for changing allowed TLS versions + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- ``s3://`` requests now use HTTPS by default, instead of plaintext HTTP. + + Previously, :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler` + sent signed S3 requests over plaintext HTTP unless + ``request.meta["is_secure"]`` was set to a true value, exposing the request + path, the AWS ``Authorization`` header, the ``X-Amz-Security-Token`` header + (when using temporary credentials), and the response contents to network + attackers, who could also tamper with responses. See the `76g3-c3x4-crvx`_ + security advisory for details. + + To restore the previous behavior for a given request, set + ``request.meta["is_secure"]`` to ``False``. + + .. _76g3-c3x4-crvx: https://github.com/scrapy/scrapy/security/advisories/GHSA-76g3-c3x4-crvx + +Deprecations +~~~~~~~~~~~~ + +- The ``DOWNLOADER_CLIENT_TLS_METHOD`` setting is deprecated. You should use + the :setting:`DOWNLOAD_TLS_MIN_VERSION` and/or + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings instead if you want to change + the TLS method selection. + (:issue:`3288`, :issue:`6546`) + +- The following spider attributes are deprecated in favor of settings: + + - ``http_user`` (use :setting:`HTTPAUTH_USER`) + + - ``http_pass`` (use :setting:`HTTPAUTH_PASS`) + + - ``http_auth_domain`` (use :setting:`HTTPAUTH_DOMAIN`) + + (:issue:`7590`) + +- The ``scrapy.commands.ScrapyCommand.help()`` method is deprecated. It was + never called by Scrapy. + (:issue:`7626`, :issue:`7633`) + +- The following TLS-related functions and constants, intended for internal + use, are deprecated: + + - ``scrapy.core.downloader.tls.METHOD_TLS`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv10`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv11`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv12`` + + - ``scrapy.core.downloader.tls.openssl_methods`` + + - ``scrapy.core.downloader.tls.DEFAULT_CIPHERS`` + + - ``scrapy.utils.ssl.ffi_buf_to_string()`` + + - ``scrapy.utils.ssl.get_temp_key_info()`` + + - ``scrapy.utils.ssl.x509name_to_string()`` + + (:issue:`6546`, :issue:`7619`, :issue:`7665`) + +- The ``CRAWLSPIDER_FOLLOW_LINKS`` setting is deprecated. You can set + ``follow=False`` in your rules to achieve the same effect. + (:issue:`7592`) + +- Instantiating + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + without a ``crawler`` argument is deprecated. + (:issue:`7655`) + +- Instantiating + :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` without a + ``settings`` argument is deprecated. + (:issue:`7664`) + +New features +~~~~~~~~~~~~ + +- Added support for HTTP/2 requests to + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. It + requires setting the new :setting:`HTTPX_HTTP2_ENABLED` setting to + ``True``. + (:issue:`7575`) + +- Added support for SOCKS proxies to + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. + (:issue:`747`, :issue:`7575`) + +- Added :setting:`DOWNLOAD_TLS_MIN_VERSION` and + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings as replacements for the + ``DOWNLOADER_CLIENT_TLS_METHOD`` setting (which is now deprecated). + Compared to the old setting, they support specifying a range of allowed + versions and support newer TLS versions. + (:issue:`4821`, :issue:`6546`) + +- Added :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS` and + :setting:`HTTPAUTH_DOMAIN` settings and :reqmeta:`http_user`, + :reqmeta:`http_pass` and :reqmeta:`http_auth_domain` meta keys as more + flexible ways to set HTTP authentication data. + (:issue:`7590`) + +- Added a :reqmeta:`verbatim_url` meta key that can be set to ``True`` to + skip request URL canonicalization. + (:issue:`7473`) + +- Added ``deny_tags`` and ``deny_attrs`` arguments to :class:`LinkExtractor + `. + (:issue:`6321`, :issue:`7679`) + +- :attr:`scrapy.Item.fields` now returns the fields in the definition order + instead of the alphabetical one. + (:issue:`7015`, :issue:`7694`) + +- Added a :setting:`RETRY_GIVE_UP_LOG_LEVEL` setting, a + :reqmeta:`give_up_log_level` meta key and a ``give_up_log_level`` argument + of the + :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function that + allow changing the log level of the message logged when the retry limit has + been reached. + (:issue:`4622`, :issue:`5297`, :issue:`7567`) + +- It's now possible to set :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` to + ``None`` to use the default ciphers of the underlying TLS implementation. + (:issue:`7499`, :issue:`7665`) + +Improvements +~~~~~~~~~~~~ + +- :class:`~scrapy.FormRequest` is no longer deprecated, only its + ``from_response()`` method is still deprecated. + (:issue:`7561`, :issue:`7671`) + +- Switched the item definition in the default project template from a + :class:`scrapy.item.Item` to a dataclass. + (:issue:`7493`, :issue:`7513`) + +- Fixed deprecation warnings with pyOpenSSL 26.3.0. + (:issue:`7619`) + +- Removed the runtime warnings for :attr:`Spider.allowed_domains + ` containing URLs or domains with ports + instead of just domains and for spider classes having a ``start_url`` + attribute instead of :class:`~scrapy.spiders.Spider.start_urls`. Please use + :doc:`scrapy-lint ` to find mistakes in your spider code + instead. + (:issue:`4421`, :issue:`7627`) + +- :func:`scrapy.utils.test.get_crawler` now disables + :setting:`TELNETCONSOLE_ENABLED` by default. + (:issue:`7644`) + +- Other code refactoring and improvements. + (:issue:`7409`, :issue:`7593`, :issue:`7594`, :issue:`7611`, :issue:`7649`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` no + longer ignores proxy credentials for redirected or retried requests. + (:issue:`7601`, :issue:`7630`) + +- :class:`~scrapy.extensions.feedexport.GCSFeedStorage` now closes the + temporary file after the upload. + (:issue:`7546`) + +- Fixed ``scrapy shell `` running a full spider crawl when there is a + spider for the requested URL. This bug was introduced in Scrapy 2.13.0. + (:issue:`7552`, :issue:`7557`) + +- The :setting:`IMAGES_STORE_S3_ACL` and :setting:`IMAGES_STORE_GCS_ACL` + settings are no longer ignored. This bug was introduced in Scrapy 2.12.0. + (:issue:`7597`, :issue:`7614`) + +- :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` now closes + the connection after making the request. + (:issue:`7602`, :issue:`7667`) + +- Removed the deprecated ``spider`` argument from the pipeline defined in the + default project template. + (:issue:`7676`) + +- Fixed ``scrapy genspider --edit`` not working. + (:issue:`7260`, :issue:`7683`) + +- When a :class:`~scrapy.crawler.Crawler` instance is passed to + :meth:`AsyncCrawlerRunner.create_crawler() + ` or + :meth:`CrawlerRunner.create_crawler() + `, settings from both classes + are now merged, previously only the settings from the + :class:`~scrapy.crawler.Crawler` instance were used. + (:issue:`1280`, :issue:`7647`) + +- Fixed several issues with cookie handling in + :func:`scrapy.utils.request.request_to_curl`. + (:issue:`7603`, :issue:`7675`, :issue:`7684`) + +- Fixed :class:`scrapy.resolver.CachingThreadedResolver` not disabling the + cache when :setting:`DNSCACHE_ENABLED` is set to ``False``. + (:issue:`7663`) + +- Fixed :func:`scrapy.utils.response.open_in_browser` not removing comments + when looking for the ```` tag. + (:issue:`7506`) + +- Fixed checking for deprecated methods in custom :setting:`ITEM_PROCESSOR` + implementations. + (:issue:`7589`) + +- Fixed :func:`scrapy.utils.url.strip_url` corrupting some URLs with + credentials. + (:issue:`7604`, :issue:`7605`) + +- :func:`scrapy.utils.misc.rel_has_nofollow` now ignores the case when + looking for "nofollow" strings. + (:issue:`7632`) + +- Fixed an exception in :class:`scrapy.utils.sitemap.Sitemap` when parsing + some malformed sitemaps. + (:issue:`7686`, :issue:`7687`) + +Documentation +~~~~~~~~~~~~~ + +- Mentioned :doc:`scrapy-lint ` in the docs. + (:issue:`4421`, :issue:`7627`) + +- Added the docs about :ref:`security considerations `. + (:issue:`7389`, :issue:`7678`) + +- Improved the :ref:`item pipeline docs `. + (:issue:`2350`, :issue:`7676`) + +- Documented which stats are collected by + :class:`~scrapy.extensions.corestats.CoreStats`. + (:issue:`7421`) + +- Switched documentation examples from using :class:`scrapy.item.Item` to + using dataclasses. + (:issue:`7493`, :issue:`7513`) + +- Added feature comparison tables to the :ref:`download handler + ` docs. + (:issue:`7575`) + +- Improved the docs for :ref:`logging settings `. + (:issue:`6909`, :issue:`7668`) + +- Documented a way to :ref:`improve startup time and memory usage + ` by using :setting:`SPIDER_MODULES`. + (:issue:`7576`, :issue:`7600`) + +- Clarified handling of the ``type`` argument of :class:`~scrapy.Selector`. + (:issue:`7704`) + +- Other documentation improvements and fixes. + (:issue:`4954`, + :issue:`6120`, + :issue:`7286`, + :issue:`7564`, + :issue:`7573`, + :issue:`7598`, + :issue:`7599`, + :issue:`7698`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Fixed deprecation warnings with pytest 9.1.0. + (:issue:`7621`) + +- Type hints improvements and fixes. + (:issue:`6958`, :issue:`7586`) + +- CI and test improvements and fixes. + (:issue:`5954`, + :issue:`7002`, + :issue:`7017`, + :issue:`7247`, + :issue:`7508`, + :issue:`7545`, + :issue:`7566`, + :issue:`7574`, + :issue:`7585`, + :issue:`7595`, + :issue:`7608`, + :issue:`7610`, + :issue:`7612`, + :issue:`7616`, + :issue:`7625`, + :issue:`7637`, + :issue:`7639`, + :issue:`7640`, + :issue:`7641`, + :issue:`7642`, + :issue:`7643`, + :issue:`7644`, + :issue:`7645`, + :issue:`7646`, + :issue:`7654`, + :issue:`7655`, + :issue:`7664`, + :issue:`7672`, + :issue:`7677`, + :issue:`7680`, + :issue:`7682`, + :issue:`7692`) + .. _release-2.16.0: Scrapy 2.16.0 (2026-05-19) @@ -711,7 +1035,7 @@ Deprecations New features ~~~~~~~~~~~~ -- Added a new setting, :setting:`REFERER_POLICIES`, to allow customizing +- Added a new setting, :setting:`REFERRER_POLICIES`, to allow customizing supported referrer policies. Bug fixes @@ -5722,7 +6046,7 @@ The following changes may impact custom priority queue classes: * A new keyword parameter has been added: ``key``. It is a string that is always an empty string for memory queues and indicates the - :setting:`JOB_DIR` value for disk queues. + :setting:`JOBDIR` value for disk queues. * The parameter for disk queues that contains data from the previous crawl, ``startprios`` or ``slot_startprios``, is now passed as a @@ -6243,8 +6567,8 @@ Backward-incompatible changes ``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly. * :class:`~scrapy.crawler.Crawler`, - :class:`CrawlerRunner.crawl ` and - :class:`CrawlerRunner.create_crawler ` + :meth:`CrawlerRunner.crawl ` and + :meth:`CrawlerRunner.create_crawler ` no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they only accept a :class:`~scrapy.spiders.Spider` subclass now. @@ -6276,7 +6600,7 @@ New features ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be :ref:`enabled ` for a significant scheduling improvement on crawls targeting multiple web domains, at the - cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) + cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`) * A new :attr:`.Request.cb_kwargs` attribute provides a cleaner way to pass keyword arguments to callback methods @@ -8755,7 +9079,7 @@ New features and settings - In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`) - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: - - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP` + - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP`` - check the documentation for more details - Added builtin caching DNS resolver (:rev:`2728`) - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 01e4bcac1..78b1ab92b 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -168,7 +168,6 @@ Use a fallback component: from scrapy.utils.misc import build_from_crawler, load_object - FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 1a151044f..17b322b5a 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -105,6 +105,9 @@ Scrapy API requires passing a Deferred to it) using the following helpers: .. autofunction:: scrapy.utils.defer.deferred_from_coro .. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f + +The following function helps with a reverse wrapping: + .. autofunction:: scrapy.utils.defer.ensure_awaitable @@ -190,7 +193,7 @@ in future Scrapy versions. The following features are not available: :class:`~scrapy.crawler.CrawlerProcess` (:class:`~scrapy.crawler.AsyncCrawlerProcess` and :class:`~scrapy.crawler.AsyncCrawlerRunner` are available) -* Twisted-specific DNS resolvers (the :setting:`DNS_RESOLVER` setting) +* Twisted-specific DNS resolvers (the :setting:`TWISTED_DNS_RESOLVER` setting) * User and 3rd-party code that requires a reactor (see :ref:`below ` for examples) @@ -315,8 +318,7 @@ implementations, :class:`~asyncio.ProactorEventLoop` (default) and :class:`~asyncio.SelectorEventLoop` works with Twisted. Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop` -automatically when you change the :setting:`TWISTED_REACTOR` setting or call -:func:`~scrapy.utils.reactor.install_reactor`. +automatically when installing the asyncio reactor. .. note:: Other libraries you use may require :class:`~asyncio.ProactorEventLoop`, e.g. because it supports diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index d0321c906..787ef842a 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -88,6 +88,8 @@ server) is, and this extension builds on that premise. Prevent specific requests from triggering slot delay adjustments ================================================================ +.. versionadded:: 2.12.0 + AutoThrottle adjusts the delay of download slots based on the latencies of responses that belong to that download slot. The only exceptions are non-200 responses, which are only taken into account to increase that delay, but diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 8d1351eb9..270d9859d 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -199,6 +199,7 @@ Global commands: * :command:`fetch` * :command:`view` * :command:`version` +* :command:`bench` Project-only commands: @@ -207,7 +208,6 @@ Project-only commands: * :command:`list` * :command:`edit` * :command:`parse` -* :command:`bench` .. command:: startproject @@ -476,7 +476,7 @@ Supported options: * ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider -* ``--a NAME=VALUE``: set spider argument (may be repeated) +* ``-a NAME=VALUE``: set spider argument (may be repeated) * ``--callback`` or ``-c``: spider method to use as callback for parsing the response diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 116aac323..8aaf0aa31 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -268,5 +268,5 @@ You can also send multiple requests in parallel: yield { "h1": response.css("h1::text").get(), "price": responses[0].css(".price::text").get(), - "price2": responses[1].css(".color::text").get(), + "color": responses[1].css(".color::text").get(), } diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst index 888bfaf08..9bbfee9e2 100644 --- a/docs/topics/download-handlers.rst +++ b/docs/topics/download-handlers.rst @@ -39,6 +39,10 @@ for additional schemes and to replace or disable default ones: "sftp": "my.download_handlers.SftpHandler", } +.. seealso:: :ref:`security-unencrypted-protocols` and + :ref:`security-local-resources`, for the security implications of the + default ``http``, ``ftp``, ``file`` and ``data`` handlers. + Replacing HTTP(S) download handlers ----------------------------------- @@ -325,13 +329,13 @@ Other limitations: HTTPX_HTTP2_ENABLED ^^^^^^^^^^^^^^^^^^^ +.. versionadded:: 2.17.0 + Default: ``False`` Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra needs to be installed if you want to enable this setting. -.. versionadded:: VERSION - Built-in non-HTTP download handlers reference ============================================= diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 0c1af5276..383f331b5 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -351,6 +351,8 @@ HttpAuthMiddleware HTTPAUTH_USER ~~~~~~~~~~~~~ +.. versionadded:: 2.17.0 + Default: ``""`` The username to use for HTTP basic authentication, applied to all requests @@ -361,6 +363,8 @@ whose URL matches :setting:`HTTPAUTH_DOMAIN`. HTTPAUTH_PASS ~~~~~~~~~~~~~ +.. versionadded:: 2.17.0 + Default: ``""`` The password to use for HTTP basic authentication. @@ -370,6 +374,8 @@ The password to use for HTTP basic authentication. HTTPAUTH_DOMAIN ~~~~~~~~~~~~~~~ +.. versionadded:: 2.17.0 + Default: ``None`` The domain (and its subdomains) to which HTTP basic authentication credentials @@ -379,6 +385,8 @@ that this risks leaking credentials to unrelated domains. This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER` or :setting:`HTTPAUTH_PASS` is set. +.. seealso:: :ref:`security-credential-leakage` + .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication @@ -497,7 +505,7 @@ Filesystem storage backend (default) * ``response_body`` - the plain response body - * ``response_headers`` - the request headers (in raw HTTP format) + * ``response_headers`` - the response headers (in raw HTTP format) * ``meta`` - some metadata of this cache resource in Python ``repr()`` format (grep-friendly format) @@ -539,7 +547,7 @@ defines the methods described below. .. method:: open_spider(spider) This method gets called after a spider has been opened for crawling. It handles - the :signal:`open_spider ` signal. + the :signal:`spider_opened` signal. :param spider: the spider which has been opened :type spider: :class:`~scrapy.Spider` object @@ -547,7 +555,7 @@ defines the methods described below. .. method:: close_spider(spider) This method gets called after a spider has been closed. It handles - the :signal:`close_spider ` signal. + the :signal:`spider_closed` signal. :param spider: the spider which has been closed :type spider: :class:`~scrapy.Spider` object @@ -806,7 +814,6 @@ HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. setting:: HTTPPROXY_ENABLED -.. setting:: HTTPPROXY_AUTH_ENCODING HTTPPROXY_ENABLED ^^^^^^^^^^^^^^^^^ @@ -815,6 +822,8 @@ Default: ``True`` Whether or not to enable the :class:`HttpProxyMiddleware`. +.. setting:: HTTPPROXY_AUTH_ENCODING + HTTPPROXY_AUTH_ENCODING ^^^^^^^^^^^^^^^^^^^^^^^ @@ -976,7 +985,7 @@ Whether the Meta Refresh middleware will be enabled. METAREFRESH_IGNORE_TAGS ^^^^^^^^^^^^^^^^^^^^^^^ -Default: ``[]`` +Default: ``["noscript"]`` Meta tags within these tags are ignored. @@ -1083,7 +1092,7 @@ Default:: 'twisted.internet.error.ConnectionDone', 'twisted.internet.error.ConnectError', 'twisted.internet.error.ConnectionLost', - IOError, + OSError, 'scrapy.core.downloader.handlers.http11.TunnelError', ] @@ -1102,6 +1111,8 @@ exception propagation, see RETRY_GIVE_UP_LOG_LEVEL ^^^^^^^^^^^^^^^^^^^^^^^ +.. versionadded:: 2.17.0 + Default: ``"ERROR"`` :ref:`Logging level ` used for the message logged when a request diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 5a399e094..30b6536c7 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -133,7 +133,7 @@ data from it depends on the type of response: .. code-block:: python - selector = Selector(data["html"]) + selector = Selector(text=data["html"]) - If the response is JavaScript, or HTML with a ``