diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..98a74f8ce --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,31 @@ + diff --git a/.github/workflows/auto-close-llm-pr.yml b/.github/workflows/auto-close-llm-pr.yml new file mode 100644 index 000000000..160b39488 --- /dev/null +++ b/.github/workflows/auto-close-llm-pr.yml @@ -0,0 +1,48 @@ +name: Auto-close LLM PRs +on: + pull_request_target: + types: [opened] +permissions: + contents: read + pull-requests: write +jobs: + close-llm-pr: + name: Close PR if marked as LLM-written + runs-on: ubuntu-latest + steps: + - name: Check PR body and close if LLM-written + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const marker = "This PR was written entirely using an LLM"; + const { owner, repo } = context.repo; + const prNumber = context.payload.pull_request && context.payload.pull_request.number; + if (!prNumber) { + console.log('No pull request number found in context; exiting.'); + return; + } + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber }); + const body = pr.body || ""; + if (body.includes(marker)) { + if (pr.state === 'closed') { + console.log(`PR #${prNumber} already closed.`); + return; + } + await github.rest.issues.addLabels({ + owner, + repo, + issue_number: prNumber, + labels: ['spam'] + }); + await github.rest.issues.createComment({ + owner, + repo, + issue_number: prNumber, + body: "Closing this PR because it contains the disclosure: \"This PR was written entirely using an LLM\"." + }); + await github.rest.pulls.update({ owner, repo, pull_number: prNumber, state: 'closed' }); + console.log(`Closed PR #${prNumber} because marker was found.`); + } else { + console.log(`Marker not found in PR #${prNumber}; nothing to do.`); + } diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 101c6648e..ed2388a59 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -17,24 +17,28 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: pylint - python-version: "3.10" env: - TOXENV: typing + TOXENV: mypy - python-version: "3.10" env: - TOXENV: typing-tests - - python-version: "3.13" # Keep in sync with .readthedocs.yml + TOXENV: mypy-tests + # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version. + - python-version: "3.14" env: TOXENV: docs - python-version: "3.13" + env: + TOXENV: docs-tests + - python-version: "3.14" env: TOXENV: twinecheck steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 @@ -50,5 +54,5 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9894c8e7c..7779bbb6b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,10 +18,10 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: - python-version: "3.13" + python-version: "3.14" - run: | python -m pip install --upgrade build python -m build diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index f061b1892..0409b3ef2 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -13,13 +13,21 @@ concurrency: jobs: tests: runs-on: macos-latest + env: + PYTEST_ADDOPTS: -n auto strategy: fail-fast: false matrix: - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + env: + - TOXENV: py + include: + - python-version: '3.14' + env: + TOXENV: no-reactor steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 @@ -27,13 +35,16 @@ jobs: python-version: ${{ matrix.python-version }} - name: Run tests + env: ${{ matrix.env }} run: | pip install -U tox - tox -e py + tox - name: Upload coverage report uses: codecov/codecov-action@v5 - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 + uses: codecov/codecov-action@v5 + with: + report_type: test_results diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 1f1ef77ca..15f25d9b8 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -13,6 +13,8 @@ concurrency: jobs: tests: runs-on: ubuntu-latest + env: + PYTEST_ADDOPTS: -n auto strategy: fail-fast: false matrix: @@ -29,45 +31,57 @@ jobs: - python-version: "3.13" env: TOXENV: py - - python-version: "3.13" + - python-version: "3.14" + env: + TOXENV: py + - python-version: "3.14" env: TOXENV: default-reactor - - python-version: pypy3.11 + - python-version: "3.14" + env: + TOXENV: no-reactor + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 env: TOXENV: pypy3 - # pinned deps + # min deps - python-version: "3.10.19" env: - TOXENV: pinned + TOXENV: min - python-version: "3.10.19" env: - TOXENV: default-reactor-pinned - - python-version: pypy3.11 - env: - TOXENV: pypy3-pinned + TOXENV: min-default-reactor - python-version: "3.10.19" env: - TOXENV: extra-deps-pinned + TOXENV: min-no-reactor + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 + env: + TOXENV: min-pypy3 - python-version: "3.10.19" env: - TOXENV: botocore-pinned + TOXENV: min-extra-deps + - python-version: "3.10.19" + env: + TOXENV: min-botocore - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: extra-deps - - python-version: pypy3.11 + - python-version: "3.14" + env: + TOXENV: no-reactor-extra-deps + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 env: TOXENV: pypy3-extra-deps - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: botocore - - python-version: "3.13" - env: - TOXENV: mitmproxy steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 @@ -75,11 +89,14 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') + if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'min') run: | sudo apt-get update sudo apt-get install libxml2-dev libxslt-dev + - name: Install mitmproxy + run: pipx install mitmproxy + - name: Run tests env: ${{ matrix.env }} run: | @@ -91,4 +108,6 @@ jobs: - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 + uses: codecov/codecov-action@v5 + with: + report_type: test_results diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 73b87f8ed..f413782bc 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -13,6 +13,8 @@ concurrency: jobs: tests: runs-on: windows-latest + env: + PYTEST_ADDOPTS: -n auto strategy: fail-fast: false matrix: @@ -29,24 +31,30 @@ jobs: - python-version: "3.13" env: TOXENV: py - - python-version: "3.13" + - python-version: "3.14" + env: + TOXENV: py + - python-version: "3.14" env: TOXENV: default-reactor + - python-version: "3.14" + env: + TOXENV: no-reactor - # pinned deps + # min deps - python-version: "3.10.11" env: - TOXENV: pinned + TOXENV: min - python-version: "3.10.11" env: - TOXENV: extra-deps-pinned + TOXENV: min-extra-deps - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: extra-deps steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v6 @@ -64,4 +72,6 @@ jobs: - name: Upload test results if: ${{ !cancelled() }} - uses: codecov/test-results-action@v1 + uses: codecov/codecov-action@v5 + with: + report_type: test_results diff --git a/.gitignore b/.gitignore index 4100bcd97..5e52ecf1e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ *.pyc _trial_temp* dropin.cache -docs/build +docs/_build *egg-info .tox/ venv/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5ffe2a8ea..311df7052 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -6,7 +6,7 @@ exclude: | ) repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.14.2 + rev: v0.15.20 hooks: - id: ruff-check args: [ --fix ] @@ -16,13 +16,17 @@ repos: hooks: - id: blacken-docs additional_dependencies: - - black==25.9.0 + - black==26.5.1 - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v1.0.0 + rev: v1.0.2 hooks: - id: sphinx-lint +- repo: https://github.com/scrapy/sphinx-scrapy + rev: 0.8.8 + hooks: + - id: sphinx-scrapy diff --git a/.readthedocs.yml b/.readthedocs.yml index 23e4cabea..a2773dcf2 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,17 +1,10 @@ version: 2 -formats: all -sphinx: - configuration: docs/conf.py - fail_on_warning: true - build: os: ubuntu-24.04 tools: - # For available versions, see: - # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.13" # Keep in sync with .github/workflows/checks.yml - -python: - install: - - requirements: docs/requirements.txt - - path: . + python: "3.14" + commands: + - pip install tox + - tox -e docs + - mkdir -p $READTHEDOCS_OUTPUT/html + - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/ diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..24a426d36 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,6 @@ +cff-version: 1.2.0 +message: If you use Scrapy in published research, please cite it as below. +title: Scrapy +authors: + - name: Scrapy contributors +url: https://scrapy.org diff --git a/SECURITY.md b/SECURITY.md index a5a5c7fb3..c1a5482c6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,8 +4,8 @@ | Version | Supported | | ------- | ------------------ | -| 2.13.x | :white_check_mark: | -| < 2.13.x | :x: | +| 2.17.x | :white_check_mark: | +| < 2.17.x | :x: | ## Reporting a Vulnerability diff --git a/conftest.py b/conftest.py index 0dd9d1ee0..7403b07b2 100644 --- a/conftest.py +++ b/conftest.py @@ -1,5 +1,6 @@ from __future__ import annotations +import importlib from pathlib import Path from typing import TYPE_CHECKING @@ -7,8 +8,10 @@ import pytest from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import set_asyncio_event_loop_policy +from scrapy.utils.reactorless import install_reactor_import_hook from tests.keys import generate_keys from tests.mockserver.http import MockServer +from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd if TYPE_CHECKING: from collections.abc import Generator @@ -21,20 +24,13 @@ def _py_files(folder): collect_ignore = [ # may need extra deps "docs/_ext", - # not a test, but looks like a test - "scrapy/utils/testproc.py", - "scrapy/utils/testsite.py", - "tests/ftpserver.py", - "tests/mockserver.py", - "tests/pipelines.py", - "tests/spiders.py", - # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerProcessSubprocess *_py_files("tests/AsyncCrawlerProcess"), - # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerRunnerSubprocess *_py_files("tests/AsyncCrawlerRunner"), - # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), - # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerRunnerSubprocess *_py_files("tests/CrawlerRunner"), ] @@ -54,6 +50,22 @@ if not H2_ENABLED: ) ) +try: + import httpx # noqa: F401 +except ImportError: + collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py") + + +def pytest_addoption(parser, pluginmanager): + if pluginmanager.hasplugin("twisted"): + return + # add the full choice set so that pytest doesn't complain about invalid choices in some cases + parser.addoption( + "--reactor", + default="none", + choices=["asyncio", "default", "none"], + ) + @pytest.fixture(scope="session") def mockserver() -> Generator[MockServer]: @@ -61,77 +73,71 @@ def mockserver() -> Generator[MockServer]: yield mockserver +@pytest.fixture # function scope because it modifies os.environ +def proxy_server( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> Generator[str]: + kind = request.param + proxy = MitmProxy(mode="socks5" if kind == "socks5" else None) + url = proxy.start() + if kind == "https": + url = url.replace("http://", "https://") + monkeypatch.setenv("http_proxy", url) + monkeypatch.setenv("https_proxy", url) + + try: + yield kind + finally: + proxy.stop() + + @pytest.fixture(scope="session") def reactor_pytest(request) -> str: return request.config.getoption("--reactor") -@pytest.fixture(autouse=True) -def only_asyncio(request, reactor_pytest): - if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio": - pytest.skip("This test is only run with --reactor=asyncio") - - -@pytest.fixture(autouse=True) -def only_not_asyncio(request, reactor_pytest): - if ( - request.node.get_closest_marker("only_not_asyncio") - and reactor_pytest == "asyncio" - ): - pytest.skip("This test is only run without --reactor=asyncio") - - -@pytest.fixture(autouse=True) -def requires_uvloop(request): - if not request.node.get_closest_marker("requires_uvloop"): - return - try: - import uvloop # noqa: PLC0415 - - del uvloop - except ImportError: - pytest.skip("uvloop is not installed") - - -@pytest.fixture(autouse=True) -def requires_botocore(request): - if not request.node.get_closest_marker("requires_botocore"): - return - try: - import botocore # noqa: PLC0415 - - del botocore - except ImportError: - pytest.skip("botocore is not installed") - - -@pytest.fixture(autouse=True) -def requires_boto3(request): - if not request.node.get_closest_marker("requires_boto3"): - return - try: - import boto3 # noqa: PLC0415 - - del boto3 - except ImportError: - pytest.skip("boto3 is not installed") - - -@pytest.fixture(autouse=True) -def requires_mitmproxy(request): - if not request.node.get_closest_marker("requires_mitmproxy"): - return - try: - import mitmproxy # noqa: F401, PLC0415 - except ImportError: - pytest.skip("mitmproxy is not installed") - - def pytest_configure(config): if config.getoption("--reactor") == "asyncio": # Needed on Windows to switch from proactor to selector for Twisted reactor compatibility. # If we decide to run tests with both, we will need to add a new option and check it here. set_asyncio_event_loop_policy() + elif config.getoption("--reactor") == "none": + install_reactor_import_hook() + + +def pytest_runtest_setup(item): + # Skip tests based on reactor markers + reactor = item.config.getoption("--reactor") + + if item.get_closest_marker("requires_reactor") and reactor == "none": + pytest.skip('This test is only run when the --reactor value is not "none"') + + if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}: + pytest.skip( + 'This test is only run when the --reactor value is "asyncio" (default) or "none"' + ) + + if item.get_closest_marker("only_not_asyncio") and reactor in {"asyncio", "none"}: + pytest.skip( + 'This test is only run when the --reactor value is not "asyncio" (default) or "none"' + ) + + # Skip tests requiring optional dependencies + optional_deps = [ + "uvloop", + "botocore", + "boto3", + ] + + for module in optional_deps: + if item.get_closest_marker(f"requires_{module}"): + try: + importlib.import_module(module) + except ImportError: + pytest.skip(f"{module} is not installed") + + if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None: + pytest.skip("mitmdump is not available") # Generate localhost certificate files, needed by some tests diff --git a/docs/README.rst b/docs/README.rst deleted file mode 100644 index 36dd5aea4..000000000 --- a/docs/README.rst +++ /dev/null @@ -1,68 +0,0 @@ -:orphan: - -====================================== -Scrapy documentation quick start guide -====================================== - -This file provides a quick guide on how to compile the Scrapy documentation. - - -Setup the environment ---------------------- - -To compile the documentation you need Sphinx Python library. To install it -and all its dependencies run the following command from this dir - -:: - - pip install -r requirements.txt - - -Compile the documentation -------------------------- - -To compile the documentation (to classic HTML output) run the following command -from this dir:: - - make html - -Documentation will be generated (in HTML format) inside the ``build/html`` dir. - - -View the documentation ----------------------- - -To view the documentation run the following command:: - - make htmlview - -This command will fire up your default browser and open the main page of your -(previously generated) HTML documentation. - - -Start over ----------- - -To clean up all generated documentation files and start from scratch run:: - - make clean - -Keep in mind that this command won't touch any documentation source files. - - -Recreating documentation on the fly ------------------------------------ - -There is a way to recreate the doc automatically when you make changes, you -need to install watchdog (``pip install watchdog``) and then use:: - - make watch - -Alternative method using tox ----------------------------- - -To compile the documentation to HTML run the following command:: - - tox -e docs - -Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir. diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 776afaa6c..edb91bfb9 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -77,6 +77,25 @@ def make_setting_element( return item +def make_setting_markdown_item( + setting_data: SettingData, app: Sphinx, fromdocname: str +) -> str: + uri = app.builder.get_relative_uri(fromdocname, setting_data["docname"]) + if uri.startswith("#"): + target = f"#{setting_data['refid']}" + else: + target = f"{uri}#{setting_data['refid']}" + return f"* [{setting_data['setting_name']}]({target})" + + +def _iter_sorted_settings(env: Any, fromdocname: str) -> list[SettingData]: + return [ + d + for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined] + if fromdocname != d["docname"] + ] + + def replace_settingslist_nodes( app: Sphinx, doctree: document, fromdocname: str ) -> None: @@ -87,13 +106,29 @@ def replace_settingslist_nodes( settings_list.extend( [ make_setting_element(d, app, fromdocname) - for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined] - if fromdocname != d["docname"] + for d in _iter_sorted_settings(env, fromdocname) ] ) node.replace_self(settings_list) +def visit_settingslist_node_markdown(translator: Any, _node: Node) -> None: + builder = translator.builder + env = builder.env + fromdocname = getattr(builder, "current_doc_name", env.docname) + lines = [ + make_setting_markdown_item(setting_data, builder.app, fromdocname) + for setting_data in _iter_sorted_settings(env, fromdocname) + ] + if lines: + translator.add("\n".join(lines), prefix_eol=2, suffix_eol=2) + raise nodes.SkipNode + + +def depart_settingslist_node_markdown(_translator: Any, _node: Node) -> None: + return None + + def source_role( name, rawtext, text: str, lineno, inliner, options=None, content=None ) -> tuple[list[Any], list[Any]]: @@ -126,34 +161,22 @@ def rev_role( return [node], [] -def setup(app: Sphinx) -> None: - app.add_crossref_type( - directivename="setting", - rolename="setting", - indextemplate="pair: %s; setting", - ) - app.add_crossref_type( - directivename="signal", - rolename="signal", - indextemplate="pair: %s; signal", - ) - app.add_crossref_type( - directivename="command", - rolename="command", - indextemplate="pair: %s; command", - ) - app.add_crossref_type( - directivename="reqmeta", - rolename="reqmeta", - indextemplate="pair: %s; reqmeta", - ) +def setup(app: Sphinx) -> dict[str, Any]: app.add_role("source", source_role) app.add_role("commit", commit_role) app.add_role("issue", issue_role) app.add_role("rev", rev_role) - app.add_node(SettingslistNode) + app.add_node( + SettingslistNode, + markdown=(visit_settingslist_node_markdown, depart_settingslist_node_markdown), + singlemarkdown=( + visit_settingslist_node_markdown, + depart_settingslist_node_markdown, + ), + ) app.add_directive("settingslist", SettingsListDirective) app.connect("doctree-read", collect_scrapy_settings_refs) app.connect("doctree-resolved", replace_settingslist_nodes) + return {"parallel_read_safe": True} diff --git a/docs/_ext/scrapyfixautodoc.py b/docs/_ext/scrapyfixautodoc.py index d7a3fb514..e342e92cf 100644 --- a/docs/_ext/scrapyfixautodoc.py +++ b/docs/_ext/scrapyfixautodoc.py @@ -3,16 +3,19 @@ Must be included after 'sphinx.ext.autodoc'. Fixes unwanted 'alias of' behavior. https://github.com/sphinx-doc/sphinx/issues/4422 """ +from typing import Any + # pylint: disable=import-error from sphinx.application import Sphinx def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) -> bool: if not skip: - # autodocs was generating a text "alias of" for the following members + # autodoc was generating the text "alias of" for the following members return name in {"default_item_class", "default_selector_class"} return skip -def setup(app: Sphinx) -> None: +def setup(app: Sphinx) -> dict[str, Any]: app.connect("autodoc-skip-member", maybe_skip_member) + return {"parallel_read_safe": True} diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html index 6ec565e24..29394799b 100644 --- a/docs/_templates/layout.html +++ b/docs/_templates/layout.html @@ -1,6 +1,6 @@ {% extends "!layout.html" %} -{# Overriden to include a link to scrapy.org, not just to the docs root #} +{# Overridden to include a link to scrapy.org, not just to the docs root #} {%- block sidebartitle %} {# the logo helper function was removed in Sphinx 6 and deprecated since Sphinx 4 #} diff --git a/docs/conf.py b/docs/conf.py index 9b1db9bb7..b950c4ee2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,11 +28,9 @@ author = "Scrapy developers" extensions = [ "notfound.extension", "scrapydocs", - "sphinx.ext.autodoc", + "sphinx_scrapy", "scrapyfixautodoc", # Must be after "sphinx.ext.autodoc" "sphinx.ext.coverage", - "sphinx.ext.intersphinx", - "sphinx.ext.viewcode", "sphinx_rtd_dark_mode", ] @@ -147,22 +145,26 @@ coverage_ignore_pyobjects = [ # -- Options for the InterSphinx extension ----------------------------------- # https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration -intersphinx_mapping = { - "attrs": ("https://www.attrs.org/en/stable/", None), - "coverage": ("https://coverage.readthedocs.io/en/latest", None), - "cryptography": ("https://cryptography.io/en/latest/", None), - "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), - "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), - "parsel": ("https://parsel.readthedocs.io/en/latest/", None), - "pytest": ("https://docs.pytest.org/en/latest", None), - "python": ("https://docs.python.org/3", None), - "sphinx": ("https://www.sphinx-doc.org/en/master", None), - "tox": ("https://tox.wiki/en/latest/", None), - "twisted": ("https://docs.twisted.org/en/stable/", None), - "twistedapi": ("https://docs.twisted.org/en/stable/api/", None), - "w3lib": ("https://w3lib.readthedocs.io/en/latest", None), -} intersphinx_disabled_reftypes: Sequence[str] = [] +# sphinx-scrapy --------------------------------------------------------------- + +scrapy_intersphinx_enable = [ + "attrs", + "coverage", + "cryptography", + "cssselect", + "form2request", + "itemloaders", + "parsel", + "pytest", + "scrapy-lint", + "sphinx", + "tox", + "twisted", + "twistedapi", + "w3lib", +] + # -- Other options ------------------------------------------------------------ default_dark_mode = False diff --git a/docs/contributing.rst b/docs/contributing.rst index 3976d34c2..6d2e08fe8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -258,7 +258,7 @@ Scrapy: * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. - See https://docs.github.com/en/get-started/getting-started-with-git/setting-your-username-in-git + See https://docs.github.com/en/get-started/git-basics/setting-your-username-in-git for setup instructions. .. _scrapy-pre-commit: @@ -323,9 +323,10 @@ deprecation removals are documented in the :ref:`release notes `. Tests ===== -Tests are implemented using the :doc:`Twisted unit-testing framework -`. Running tests requires -:doc:`tox `. +Tests are implemented using pytest_. Running tests requires :doc:`tox +`. + +.. _pytest: https://pytest.org .. _running-tests: @@ -371,6 +372,21 @@ To see coverage report install :doc:`coverage ` see output of ``coverage --help`` for more options like html or xml report. +Some tests need a ``mitmdump`` executable (from mitmproxy_) to test against a +fully featured proxy server; they are skipped when one cannot be found +(``mitmproxy`` is intentionally not a test dependency that would be installed +into test venvs, as that sometimes leads to various dependency conflicts). +To run these tests, make ``mitmdump`` available in one of these ways: + +* install ``mitmproxy`` so that ``mitmdump`` is on your ``PATH``, e.g. with + pipx_ (``pipx install mitmproxy``) or uv_ (``uv tool install mitmproxy``); + +* have uv_ installed, in which case the tests will run + ``uvx --from mitmproxy mitmdump``; + +* set the ``MITMDUMP`` environment variable to the path of a ``mitmdump`` + executable. + Writing tests ------------- @@ -390,8 +406,7 @@ And their unit-tests are in:: .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users -.. _Scrapy subreddit: https://reddit.com/r/scrapy -.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS +.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/ .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues .. _PEP 257: https://peps.python.org/pep-0257/ @@ -399,3 +414,6 @@ And their unit-tests are in:: .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 .. _test coverage: https://app.codecov.io/gh/scrapy/scrapy +.. _mitmproxy: https://mitmproxy.org/ +.. _pipx: https://pipx.pypa.io/ +.. _uv: https://docs.astral.sh/uv/ diff --git a/docs/faq.rst b/docs/faq.rst index fdf0582de..8f2013581 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -82,10 +82,18 @@ to steal from us! Does Scrapy work with HTTP proxies? ----------------------------------- -Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP -Proxy downloader middleware. See +Yes. Support for HTTP proxies is provided through the HTTP Proxy downloader +middleware. See :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`. +Does Scrapy work with SOCKS proxies? +------------------------------------ + +Yes, when using +:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. See +:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and the +handler documentation. + How can I scrape an item with attributes in different pages? ------------------------------------------------------------ @@ -277,7 +285,8 @@ consume a lot of memory. In order to avoid parsing all the entire feed at once in memory, you can use the :func:`~scrapy.utils.iterators.xmliter_lxml` and :func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what -:class:`~scrapy.spiders.XMLFeedSpider` uses. +:class:`~scrapy.spiders.XMLFeedSpider` and +:class:`~scrapy.spiders.CSVFeedSpider` use. .. autofunction:: scrapy.utils.iterators.xmliter_lxml @@ -352,15 +361,19 @@ method for this purpose. For example: def process_spider_output(self, response, result): for item_or_request in result: if isinstance(item_or_request, Request): + yield item_or_request continue - adapter = ItemAdapter(item) + adapter = ItemAdapter(item_or_request) for _ in range(adapter["multiply_by"]): - yield deepcopy(item) + yield deepcopy(item_or_request) Does Scrapy support IPv6 addresses? ----------------------------------- -Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. +Yes, but when using +:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` or +:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` you need to +set :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. Note that by doing so, you lose the ability to set a specific timeout for DNS requests (the value of the :setting:`DNS_TIMEOUT` setting is ignored). @@ -371,8 +384,9 @@ How to deal with ``: filedescriptor out of range in select() ---------------------------------------------------------------------------------------------- This issue `has been reported`_ to appear when running broad crawls in macOS, where the default -Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a -different reactor is possible by using the :setting:`TWISTED_REACTOR` setting. +Twisted reactor was :class:`twisted.internet.selectreactor.SelectReactor` at that time. +If you have switched to this reactor using the :setting:`TWISTED_REACTOR` setting you can switch +to a different one in the same way. .. _faq-stop-response-download: @@ -398,7 +412,6 @@ How can I make a blank request? from scrapy import Request - blank_request = Request("data:,") In this case, the URL is set to a data URI scheme. Data URLs allow you to include data @@ -418,4 +431,3 @@ See :issue:`2680`. .. _has been reported: https://github.com/scrapy/scrapy/issues/2905 .. _Python standard library modules: https://docs.python.org/3/py-modindex.html .. _Python package: https://pypi.org/ -.. _user agents: https://en.wikipedia.org/wiki/User_agent diff --git a/docs/index.rst b/docs/index.rst index 71047f9ef..688cab81b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ Having trouble? We'd like to help! * Ask or search questions in `StackOverflow using the scrapy tag`_. * Ask or search questions in the `Scrapy subreddit`_. * Search for questions on the archives of the `scrapy-users mailing list`_. -* Ask a question in the `#scrapy IRC channel`_, +* Ask a question in the `#scrapy IRC channel`_. * Report bugs with Scrapy in our `issue tracker`_. * Join the Discord community `Scrapy Discord`_. @@ -91,15 +91,15 @@ Basic concepts :doc:`topics/selectors` Extract the data from web pages using XPath. -:doc:`topics/shell` - Test your extraction code in an interactive environment. - :doc:`topics/items` Define the data you want to scrape. :doc:`topics/loaders` Populate your items with the extracted data. +:doc:`topics/shell` + Test your extraction code in an interactive environment. + :doc:`topics/item-pipeline` Post-process and store your scraped data. @@ -128,7 +128,6 @@ Built-in services topics/logging topics/stats - topics/email topics/telnetconsole :doc:`topics/logging` @@ -137,9 +136,6 @@ Built-in services :doc:`topics/stats` Collect statistics about your scraping crawler. -:doc:`topics/email` - Send email notifications when certain events occur. - :doc:`topics/telnetconsole` Inspect a running crawler using a built-in Python console. @@ -155,6 +151,7 @@ Solving specific problems topics/debug topics/contracts topics/practices + topics/security topics/broad-crawls topics/developer-tools topics/dynamic-content @@ -179,6 +176,10 @@ Solving specific problems :doc:`topics/practices` Get familiar with some Scrapy common practices. +:doc:`topics/security` + Understand the security implications of Scrapy defaults and how to harden + them. + :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. @@ -229,6 +230,7 @@ Extending Scrapy topics/signals topics/scheduler topics/exporters + topics/download-handlers topics/components topics/api @@ -257,6 +259,9 @@ Extending Scrapy :doc:`topics/exporters` Quickly export your scraped items to a file (XML, CSV, etc). +:doc:`topics/download-handlers` + Customize how requests are downloaded or add support for new URL schemes. + :doc:`topics/components` Learn the common API and some good practices when building custom Scrapy components. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 0810e7274..67a84e538 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -230,8 +230,8 @@ Installing Scrapy with PyPy on Windows is not tested. You can check that Scrapy is installed correctly by running ``scrapy bench``. If this command gives errors such as ``TypeError: ... got 2 unexpected keyword arguments``, this means -that setuptools was unable to pick up one PyPy-specific dependency. -To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``. +that the ``PyPyDispatcher`` dependency wasn't installed. To fix this issue, run +``pip install 'PyPyDispatcher>=2.1.0'``. .. _intro-install-troubleshooting: @@ -263,7 +263,6 @@ reinstall Twisted with the :code:`tls` extra option:: For details, see `Issue #2473 `_. .. _Python: https://www.python.org/ -.. _pip: https://pip.pypa.io/en/latest/installing/ .. _lxml: https://lxml.de/index.html .. _parsel: https://pypi.org/project/parsel/ .. _w3lib: https://pypi.org/project/w3lib/ @@ -273,8 +272,7 @@ For details, see `Issue #2473 `_. .. _setuptools: https://pypi.org/pypi/setuptools .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ -.. _Anaconda: https://docs.anaconda.com/anaconda/ +.. _Anaconda: https://www.anaconda.com/docs/main .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html -.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio .. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/ .. _conda-forge: https://conda-forge.org/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index d05e46551..5937c5860 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -83,16 +83,17 @@ While this enables you to do very fast crawls (sending multiple concurrent requests at the same time, in a fault-tolerant way) Scrapy also gives you control over the politeness of the crawl through :ref:`a few settings `. You can do things like setting a download delay between -each request, limiting the amount of concurrent requests per domain or per IP, and +each request, limiting the amount of concurrent requests per domain, and even :ref:`using an auto-throttling extension ` that tries to figure these settings out automatically. .. note:: This is using :ref:`feed exports ` to generate the - JSON file, you can easily change the export format (XML or CSV, for example) or the - storage backend (FTP or `Amazon S3`_, for example). You can also write an - :ref:`item pipeline ` to store the items in a database. + JSON Lines file, you can easily change the export format (XML or CSV, for + example) or the storage backend (FTP or `Amazon S3`_, for example). You can + also write an :ref:`item pipeline ` to store the + items in a database. .. _topics-whatelse: @@ -150,7 +151,7 @@ The next steps for you are to :ref:`install Scrapy `, a full-blown Scrapy project and `join the community`_. Thanks for your interest! -.. _join the community: https://scrapy.org/community/ +.. _join the community: https://www.scrapy.org/community .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping .. _Amazon Associates Web Services: https://affiliate-program.amazon.com/welcome/ecs .. _Amazon S3: https://aws.amazon.com/s3/ diff --git a/docs/news.rst b/docs/news.rst index 96e5cb2b1..63f0d69e4 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,8 +3,1138 @@ Release notes ============= -Scrapy VERSION (unreleased) ---------------------------- +.. _release-2.17.0: + +Scrapy 2.17.0 (2026-07-07) +-------------------------- + +Highlights: + +- Security bug fixes + +- HTTP/2 and SOCKS proxy support for ``HttpxDownloadHandler`` + +- Improved settings for changing allowed TLS versions + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- ``s3://`` requests now use HTTPS by default, instead of plaintext HTTP. + + Previously, :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler` + sent signed S3 requests over plaintext HTTP unless + ``request.meta["is_secure"]`` was set to a true value, exposing the request + path, the AWS ``Authorization`` header, the ``X-Amz-Security-Token`` header + (when using temporary credentials), and the response contents to network + attackers, who could also tamper with responses. See the `76g3-c3x4-crvx`_ + security advisory for details. + + To restore the previous behavior for a given request, set + ``request.meta["is_secure"]`` to ``False``. + + .. _76g3-c3x4-crvx: https://github.com/scrapy/scrapy/security/advisories/GHSA-76g3-c3x4-crvx + +Deprecations +~~~~~~~~~~~~ + +- The ``DOWNLOADER_CLIENT_TLS_METHOD`` setting is deprecated. You should use + the :setting:`DOWNLOAD_TLS_MIN_VERSION` and/or + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings instead if you want to change + the TLS method selection. + (:issue:`3288`, :issue:`6546`) + +- The following spider attributes are deprecated in favor of settings: + + - ``http_user`` (use :setting:`HTTPAUTH_USER`) + + - ``http_pass`` (use :setting:`HTTPAUTH_PASS`) + + - ``http_auth_domain`` (use :setting:`HTTPAUTH_DOMAIN`) + + (:issue:`7590`) + +- The ``scrapy.commands.ScrapyCommand.help()`` method is deprecated. It was + never called by Scrapy. + (:issue:`7626`, :issue:`7633`) + +- The following TLS-related functions and constants, intended for internal + use, are deprecated: + + - ``scrapy.core.downloader.tls.METHOD_TLS`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv10`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv11`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv12`` + + - ``scrapy.core.downloader.tls.openssl_methods`` + + - ``scrapy.core.downloader.tls.DEFAULT_CIPHERS`` + + - ``scrapy.utils.ssl.ffi_buf_to_string()`` + + - ``scrapy.utils.ssl.get_temp_key_info()`` + + - ``scrapy.utils.ssl.x509name_to_string()`` + + (:issue:`6546`, :issue:`7619`, :issue:`7665`) + +- The ``CRAWLSPIDER_FOLLOW_LINKS`` setting is deprecated. You can set + ``follow=False`` in your rules to achieve the same effect. + (:issue:`7592`) + +- Instantiating + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + without a ``crawler`` argument is deprecated. + (:issue:`7655`) + +- Instantiating + :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` without a + ``settings`` argument is deprecated. + (:issue:`7664`) + +New features +~~~~~~~~~~~~ + +- Added support for HTTP/2 requests to + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. It + requires setting the new :setting:`HTTPX_HTTP2_ENABLED` setting to + ``True``. + (:issue:`7575`) + +- Added support for SOCKS proxies to + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. + (:issue:`747`, :issue:`7575`) + +- Added :setting:`DOWNLOAD_TLS_MIN_VERSION` and + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings as replacements for the + ``DOWNLOADER_CLIENT_TLS_METHOD`` setting (which is now deprecated). + Compared to the old setting, they support specifying a range of allowed + versions and support newer TLS versions. + (:issue:`4821`, :issue:`6546`) + +- Added :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS` and + :setting:`HTTPAUTH_DOMAIN` settings and :reqmeta:`http_user`, + :reqmeta:`http_pass` and :reqmeta:`http_auth_domain` meta keys as more + flexible ways to set HTTP authentication data. + (:issue:`7590`) + +- Added a :reqmeta:`verbatim_url` meta key that can be set to ``True`` to + skip request URL canonicalization. + (:issue:`7473`) + +- Added ``deny_tags`` and ``deny_attrs`` arguments to :class:`LinkExtractor + `. + (:issue:`6321`, :issue:`7679`) + +- :attr:`scrapy.Item.fields` now returns the fields in the definition order + instead of the alphabetical one. + (:issue:`7015`, :issue:`7694`) + +- Added a :setting:`RETRY_GIVE_UP_LOG_LEVEL` setting, a + :reqmeta:`give_up_log_level` meta key and a ``give_up_log_level`` argument + of the + :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function that + allow changing the log level of the message logged when the retry limit has + been reached. + (:issue:`4622`, :issue:`5297`, :issue:`7567`) + +- It's now possible to set :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` to + ``None`` to use the default ciphers of the underlying TLS implementation. + (:issue:`7499`, :issue:`7665`) + +Improvements +~~~~~~~~~~~~ + +- :class:`~scrapy.FormRequest` is no longer deprecated, only its + ``from_response()`` method is still deprecated. + (:issue:`7561`, :issue:`7671`) + +- Switched the item definition in the default project template from a + :class:`scrapy.item.Item` to a dataclass. + (:issue:`7493`, :issue:`7513`) + +- Fixed deprecation warnings with pyOpenSSL 26.3.0. + (:issue:`7619`) + +- Removed the runtime warnings for :attr:`Spider.allowed_domains + ` containing URLs or domains with ports + instead of just domains and for spider classes having a ``start_url`` + attribute instead of :class:`~scrapy.spiders.Spider.start_urls`. Please use + :doc:`scrapy-lint ` to find mistakes in your spider code + instead. + (:issue:`4421`, :issue:`7627`) + +- :func:`scrapy.utils.test.get_crawler` now disables + :setting:`TELNETCONSOLE_ENABLED` by default. + (:issue:`7644`) + +- Other code refactoring and improvements. + (:issue:`7409`, :issue:`7593`, :issue:`7594`, :issue:`7611`, :issue:`7649`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` no + longer ignores proxy credentials for redirected or retried requests. + (:issue:`7601`, :issue:`7630`) + +- :class:`~scrapy.extensions.feedexport.GCSFeedStorage` now closes the + temporary file after the upload. + (:issue:`7546`) + +- Fixed ``scrapy shell `` running a full spider crawl when there is a + spider for the requested URL. This bug was introduced in Scrapy 2.13.0. + (:issue:`7552`, :issue:`7557`) + +- The :setting:`IMAGES_STORE_S3_ACL` and :setting:`IMAGES_STORE_GCS_ACL` + settings are no longer ignored. This bug was introduced in Scrapy 2.12.0. + (:issue:`7597`, :issue:`7614`) + +- :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` now closes + the connection after making the request. + (:issue:`7602`, :issue:`7667`) + +- Removed the deprecated ``spider`` argument from the pipeline defined in the + default project template. + (:issue:`7676`) + +- Fixed ``scrapy genspider --edit`` not working. + (:issue:`7260`, :issue:`7683`) + +- When a :class:`~scrapy.crawler.Crawler` instance is passed to + :meth:`AsyncCrawlerRunner.create_crawler() + ` or + :meth:`CrawlerRunner.create_crawler() + `, settings from both classes + are now merged, previously only the settings from the + :class:`~scrapy.crawler.Crawler` instance were used. + (:issue:`1280`, :issue:`7647`) + +- Fixed several issues with cookie handling in + :func:`scrapy.utils.request.request_to_curl`. + (:issue:`7603`, :issue:`7675`, :issue:`7684`) + +- Fixed :class:`scrapy.resolver.CachingThreadedResolver` not disabling the + cache when :setting:`DNSCACHE_ENABLED` is set to ``False``. + (:issue:`7663`) + +- Fixed :func:`scrapy.utils.response.open_in_browser` not removing comments + when looking for the ```` tag. + (:issue:`7506`) + +- Fixed checking for deprecated methods in custom :setting:`ITEM_PROCESSOR` + implementations. + (:issue:`7589`) + +- Fixed :func:`scrapy.utils.url.strip_url` corrupting some URLs with + credentials. + (:issue:`7604`, :issue:`7605`) + +- :func:`scrapy.utils.misc.rel_has_nofollow` now ignores the case when + looking for "nofollow" strings. + (:issue:`7632`) + +- Fixed an exception in :class:`scrapy.utils.sitemap.Sitemap` when parsing + some malformed sitemaps. + (:issue:`7686`, :issue:`7687`) + +Documentation +~~~~~~~~~~~~~ + +- Mentioned :doc:`scrapy-lint ` in the docs. + (:issue:`4421`, :issue:`7627`) + +- Added the docs about :ref:`security considerations `. + (:issue:`7389`, :issue:`7678`) + +- Improved the :ref:`item pipeline docs `. + (:issue:`2350`, :issue:`7676`) + +- Documented which stats are collected by + :class:`~scrapy.extensions.corestats.CoreStats`. + (:issue:`7421`) + +- Switched documentation examples from using :class:`scrapy.item.Item` to + using dataclasses. + (:issue:`7493`, :issue:`7513`) + +- Added feature comparison tables to the :ref:`download handler + ` docs. + (:issue:`7575`) + +- Improved the docs for :ref:`logging settings `. + (:issue:`6909`, :issue:`7668`) + +- Documented a way to :ref:`improve startup time and memory usage + ` by using :setting:`SPIDER_MODULES`. + (:issue:`7576`, :issue:`7600`) + +- Clarified handling of the ``type`` argument of :class:`~scrapy.Selector`. + (:issue:`7704`) + +- Other documentation improvements and fixes. + (:issue:`4954`, + :issue:`6120`, + :issue:`7286`, + :issue:`7564`, + :issue:`7573`, + :issue:`7598`, + :issue:`7599`, + :issue:`7698`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Fixed deprecation warnings with pytest 9.1.0. + (:issue:`7621`) + +- Type hints improvements and fixes. + (:issue:`6958`, :issue:`7586`) + +- CI and test improvements and fixes. + (:issue:`5954`, + :issue:`7002`, + :issue:`7017`, + :issue:`7247`, + :issue:`7508`, + :issue:`7545`, + :issue:`7566`, + :issue:`7574`, + :issue:`7585`, + :issue:`7595`, + :issue:`7608`, + :issue:`7610`, + :issue:`7612`, + :issue:`7616`, + :issue:`7625`, + :issue:`7637`, + :issue:`7639`, + :issue:`7640`, + :issue:`7641`, + :issue:`7642`, + :issue:`7643`, + :issue:`7644`, + :issue:`7645`, + :issue:`7646`, + :issue:`7654`, + :issue:`7655`, + :issue:`7664`, + :issue:`7672`, + :issue:`7677`, + :issue:`7680`, + :issue:`7682`, + :issue:`7692`) + +.. _release-2.16.0: + +Scrapy 2.16.0 (2026-05-19) +-------------------------- + +Highlights: + +- Official support for Python 3.14 + +- Support for Twisted 26.4.0+ + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Increased the minimum versions of the following dependencies: + + - service_identity_: 18.1.0 → 23.1.0 + + (:issue:`7347`) + +- Added support for Twisted 26.4.0+. + (:issue:`7347`, :issue:`7505`, :issue:`7520`) + +- Added support for Python 3.14. + (:issue:`6604`, :issue:`7460`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The following classes and functions, intended for internal use by + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + and :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`, have + been made private: + + - ``scrapy.core.downloader.handlers.http11.ScrapyAgent`` + + - ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingTCP4ClientEndpoint`` + + - ``scrapy.core.downloader.handlers.http11.tunnel_request_data()`` + + - ``scrapy.core.downloader.handlers.http2.ScrapyH2Agent`` + + (:issue:`7496`, :issue:`7510`) + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.FormRequest`` is deprecated. You can use the :doc:`form2request + ` library instead, see :ref:`form`. + (:issue:`6438`) + +- ``scrapy.utils.python.MutableChain`` is deprecated. + (:issue:`7504`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The ``start_requests()`` method of :class:`~scrapy.Spider`, deprecated in + 2.13.0, is removed and no longer called. Use :meth:`~scrapy.Spider.start` + instead, or both to maintain support for lower Scrapy versions. + (:issue:`7490`) + +- Support for ``process_start_requests()`` methods of :ref:`spider middlewares + `, deprecated in 2.13.0, is removed. Use + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, + or both to maintain support for lower Scrapy versions. + (:issue:`7490`) + +- Support for synchronous ``process_spider_output()`` methods of spider + middlewares, deprecated in Scrapy 2.13.0, is removed. You should upgrade + the affected middlewares to have asynchronous ``process_spider_output()`` + methods. + (:issue:`7504`) + +- The ``spider`` arguments of the following methods of + :class:`~scrapy.core.scraper.Scraper`, deprecated in Scrapy 2.13.0, are + removed: + + - ``close_spider()`` + + - ``enqueue_scrape()`` + + - ``handle_spider_error()`` + + - ``handle_spider_output()`` + + (:issue:`7487`) + +- HTTP/1.0 support code, deprecated in Scrapy 2.13.0, is removed. This + includes: + + - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler`` + + - The ``scrapy.core.downloader.webclient`` module. + + - The ``DOWNLOADER_HTTPCLIENTFACTORY`` setting. + + (:issue:`7486`) + +- The following functions, deprecated in Scrapy 2.13.0, are removed, you + should import them from :mod:`w3lib.url` directly instead: + + - ``scrapy.utils.url.add_or_replace_parameter()`` + + - ``scrapy.utils.url.add_or_replace_parameters()`` + + - ``scrapy.utils.url.any_to_uri()`` + + - ``scrapy.utils.url.canonicalize_url()`` + + - ``scrapy.utils.url.file_uri_to_path()`` + + - ``scrapy.utils.url.is_url()`` + + - ``scrapy.utils.url.parse_data_uri()`` + + - ``scrapy.utils.url.parse_url()`` + + - ``scrapy.utils.url.path_to_file_uri()`` + + - ``scrapy.utils.url.safe_download_url()`` + + - ``scrapy.utils.url.safe_url_string()`` + + - ``scrapy.utils.url.url_query_cleaner()`` + + - ``scrapy.utils.url.url_query_parameter()`` + + (:issue:`7487`) + +- The following test-related code, deprecated in Scrapy 2.13.0, is removed: + + - the ``scrapy.utils.testproc`` module + + - the ``scrapy.utils.testsite`` module + + - ``scrapy.utils.test.assert_gcs_environ()`` + + - ``scrapy.utils.test.get_ftp_content_and_delete()`` + + - ``scrapy.utils.test.get_gcs_content_and_delete()`` + + - ``scrapy.utils.test.mock_google_cloud_storage()`` + + - ``scrapy.utils.test.skip_if_no_boto()`` + + - ``scrapy.utils.test.TestSpider`` + + (:issue:`7487`) + +- ``scrapy.utils.versions.scrapy_components_versions()``, deprecated in + Scrapy 2.13.0, is removed, you can use + :func:`scrapy.utils.versions.get_versions` instead. + (:issue:`7487`) + +- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` and + ``scrapy.utils.url.escape_ajax()``, deprecated in Scrapy 2.13.0, are + removed. + (:issue:`7487`) + +- The ``__init__()`` method of priority queue classes (see + :setting:`SCHEDULER_PRIORITY_QUEUE`) now needs to support a keyword-only + ``start_queue_cls`` parameter, not supporting it was deprecated in Scrapy + 2.13.0. + (:issue:`7487`) + +- ``scrapy.spiders.init.InitSpider``, deprecated in Scrapy 2.13.0, is + removed. + (:issue:`7487`) + +New features +~~~~~~~~~~~~ + +- New features and improvements for + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`: + + - Support for proxies. + + - Support for the :reqmeta:`download_latency` meta key. + + - Support for :attr:`Response.certificate + `. + + - Default headers set by the ``httpx`` library are no longer added to + requests. + + (:issue:`7441`, :issue:`7524`) + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now + skips HTTPS proxy certificate verification when the + :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting is set to ``False``. + (:issue:`7496`) + +Improvements +~~~~~~~~~~~~ + +- :func:`time.monotonic` is used instead of :func:`time.time` to calculate + elapsed time in various places. + (:issue:`7377`) + +- Improved extraction of the file extension from the URL in + :class:`~scrapy.pipelines.files.FilesPipeline`. + (:issue:`4225`, :issue:`7414`) + +- Other code refactoring and improvements. + (:issue:`7401`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now + raises an exception when a request has an ``https://`` destination and an + ``https://`` proxy, which is not supported by this handler. Previously it + tried to connect to the proxy via HTTP in this case. + (:issue:`7496`) + +- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now + raises an exception for requests with ``http://`` URLs instead of trying to + connect, which is not supported by this handler. + (:issue:`7496`) + +- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` no longer + adds the ``:status`` pseudo-header to :attr:`Response.headers + `. + (:issue:`7441`) + +- Fixed :func:`scrapy.utils.response.open_in_browser` removing the ```` + tag when adding the ```` tag. + (:issue:`7459`) + +Documentation +~~~~~~~~~~~~~ + +- Documented that + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + doesn't support HTTPS proxies for HTTPS destinations and that + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` doesn't + support proxies at all. + (:issue:`7496`) + +- Added an example of using + :class:`logging.handlers.TimedRotatingFileHandler` to rotate Scrapy logs. + (:issue:`3628`, :issue:`7501`) + +- Added a ``CITATION.cff`` file. + (:issue:`7502`, :issue:`7519`) + +- Mentioned ``DOWNLOADER_CLIENT_TLS_METHOD`` in :ref:`bans`. + (:issue:`5232`, :issue:`7518`) + +- Other documentation improvements and fixes. + (:issue:`7417`, + :issue:`7463`, + :issue:`7472`, + :issue:`7480`, + :issue:`7489`, + :issue:`7503`, + :issue:`7507`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added tests that connect to https://books.toscrape.com/ to test the + behavior with a real website. These tests are marked with the + ``requires_internet`` pytest mark and can be skipped with e.g. + ``-m 'not requires_internet'`` if you cannot or don't want to run them. + (:issue:`7520`) + +- Type hints improvements and fixes. + (:issue:`7492`, :issue:`7532`) + +- CI and test improvements and fixes. + (:issue:`7441`, :issue:`7466`, :issue:`7491`, :issue:`7496`) + +.. _release-2.15.2: + +Scrapy 2.15.2 (2026-04-28) +-------------------------- + +Bug fixes +~~~~~~~~~ + +- Fixed links in https://docs.scrapy.org/llms.txt (:issue:`7467`) + +.. _release-2.15.1: + +Scrapy 2.15.1 (2026-04-23) +-------------------------- + +Bug fixes +~~~~~~~~~ + +- Sharing of the SSL context between multiple connections, introduced in + Scrapy 2.15.0, is reverted as it caused problems and wasn't actually + needed. + (:issue:`7445`, :issue:`7450`) + +- Fixed :meth:`scrapy.settings.BaseSettings.getwithbase` failing on keys with + dots that aren't import names. It now works the way it worked before Scrapy + 2.15.0, without trying to match class objects and import path. A separate + method, + :func:`~scrapy.settings.BaseSettings.get_component_priority_dict_with_base`, + was added that does that, and it is now used for :ref:`component priority + dictionaries `. + (:issue:`7426`, :issue:`7449`) + +- Documentation rendering improvements. + (:issue:`7452`, :issue:`7454`) + +.. _release-2.15.0: + +Scrapy 2.15.0 (2026-04-09) +-------------------------- + +Highlights: + +- Experimental support for running without a Twisted reactor + +- Experimental ``httpx``-based download handler + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The built-in HTTP :ref:`download handlers ` now + raise Scrapy-specific exceptions instead of implementation-specific ones, + see :ref:`download-handlers-exceptions`. This can affect user code that + handles downloader exceptions, such as ``process_exception()`` methods of + custom :ref:`downloader middlewares `. + (:issue:`7208`) + +- In order to fix a long-standing bug with handling of asynchronous storages, + the following changes were made to media pipeline classes, which can impact + some of the user code that subclasses them or calls their methods directly: + + - overrides of :meth:`scrapy.pipelines.media.MediaPipeline.media_downloaded` + and :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` can now + return coroutines + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded`, + :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` and + :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded` now + return coroutines + + (:issue:`2183`, :issue:`6369`, :issue:`7182`) + +- ``Request`` and ``Response`` objects: ``__slots__`` and setter changes: + + - :class:`scrapy.http.Request` and :class:`scrapy.http.Response` now + define ``__slots__``. Assigning arbitrary attributes to instances (for + example, ``response.foo = 1``) will raise ``AttributeError``. Store + per-request/response data in the request/response ``meta`` mapping + instead of attaching new attributes to the objects. + + - If you maintain custom ``Request`` or ``Response`` subclasses that + relied on dynamic instance attributes, either add ``'__dict__'`` to + your subclass ``__slots__`` to allow dynamic attributes, or migrate + per-instance state to ``meta`` or explicit documented attributes. + + - The setters for ``headers``, ``flags`` and ``cookies`` no longer coerce + falsy values into ``None``. For example, ``request.headers = {}`` now + stores an empty :class:`scrapy.http.headers.Headers` instance (not + ``None``), and ``request.flags = []`` remains an empty list instead of + being set to ``None``. Update code that relied on ``is None`` checks or + the previous coercion behaviour. + + (:issue:`7036`, :issue:`7367`, :issue:`7374`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The context factory class set as the value of the + ``DOWNLOADER_CLIENTCONTEXTFACTORY`` setting is now required to support the + ``method`` argument of ``__init__()``, recommended since Scrapy 1.2.0. + (:issue:`7353`) + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.mail.MailSender`` is deprecated. Please use :mod:`smtplib`, + :mod:`twisted.mail.smtp` or other 3rd party email libraries. + (:issue:`7249`, :issue:`7263`) + +- The ``scrapy.extensions.statsmailer.StatsMailer`` extension is deprecated. + You can instead implement your own notifications by handling the + :signal:`spider_closed` signal. + (:issue:`7249`, :issue:`7263`) + +- The ``MEMUSAGE_NOTIFY_MAIL`` setting is deprecated. You can instead + implement your own notifications by handling the + :signal:`memusage_warning_reached` and :signal:`spider_closed` signals. + (:issue:`7249`, :issue:`7263`) + +- The ``DNS_RESOLVER`` setting was renamed to :setting:`TWISTED_DNS_RESOLVER` + and the old name is deprecated. + (:issue:`7350`, :issue:`7361`) + +- The ``DOWNLOADER_CLIENTCONTEXTFACTORY`` setting is deprecated. If you were + using it to switch to + ``scrapy.core.downloader.contextfactory.BrowserLikeContextFactory``, please + use the new :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting instead. If you + cannot use the default context factory for some other reason, please + subclass the :ref:`download handler ` instead. + (:issue:`7352`, :issue:`7379`) + +- ``scrapy.core.downloader.contextfactory.BrowserLikeContextFactory`` is + deprecated. You can set the new :setting:`DOWNLOAD_VERIFY_CERTIFICATES` + setting to ``True`` instead. + (:issue:`7379`) + +- The following implementation details of the context factory handling code + are deprecated: + + - ``scrapy.core.downloader.contextfactory.AcceptableProtocolsContextFactory`` + + - ``scrapy.core.downloader.contextfactory.load_context_factory_from_settings()`` + + - ``scrapy.core.downloader.contextfactory.ScrapyClientContextFactory`` + + - ``scrapy.core.downloader.tls.ScrapyClientTLSOptions`` + + (:issue:`7353`, :issue:`7391`) + +- Passing :class:`str` instead of :class:`bytes` to + :class:`scrapy.utils.sitemap.Sitemap` and + :func:`scrapy.utils.sitemap.sitemap_urls_from_robots` is deprecated. + (:issue:`7007`) + +- ``scrapy.utils.misc.walk_modules()`` is deprecated. You can use + :func:`scrapy.utils.misc.walk_modules_iter` instead. + (:issue:`7388`) + +- ``scrapy.shell.Shell.inthread`` is deprecated. You can use + :attr:`scrapy.shell.Shell.fetch_available` instead to check if + :func:`~scrapy.shell.Shell.fetch` can be used. + (:issue:`7395`) + +- ``scrapy.commands.ScrapyCommand.set_crawler()`` is deprecated. + (:issue:`7276`) + +New features +~~~~~~~~~~~~ + +- Added an *experimental* mode for running Scrapy without installing a + Twisted reactor: set :setting:`TWISTED_REACTOR_ENABLED` to ``False`` to + enable it. This mode has limitations, refer to :ref:`its documentation + ` for details. As long as it's experimental, its + behavior and related features and APIs may change in future Scrapy releases + in a breaking way. + (:issue:`6219`, + :issue:`7185`, + :issue:`7186`, + :issue:`7187`, + :issue:`7188`, + :issue:`7190`, + :issue:`7197`, + :issue:`7199`, + :issue:`7209`, + :issue:`7228`, + :issue:`7355`, + :issue:`7366`, + :issue:`7385`, + :issue:`7395`) + +- Added the :func:`scrapy.utils.reactorless.is_reactorless` function that + checks if there is a running asyncio event loop but no Twisted reactor. + (:issue:`7185`, :issue:`7199`) + +- Changed :func:`scrapy.utils.asyncio.is_asyncio_available` to return + ``True`` if there is a running asyncio loop, even if no Twisted reactor is + installed. + (:issue:`7185`, :issue:`7199`) + +- Added an *experimental* download handler that uses the httpx_ library and + doesn't require a Twisted reactor: + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. As + long as it's experimental, its behavior may change in future Scrapy + releases in a breaking way. + (:issue:`6805`, :issue:`7239`, :issue:`7368`, :issue:`7384`) + + .. _httpx: https://www.python-httpx.org/ + +- Added the :setting:`DOWNLOAD_BIND_ADDRESS` setting as a global counterpart + to the per-request :reqmeta:`bindaddress` meta key. + (:issue:`7266`, :issue:`7283`) + +- Added the :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting that can be set + to ``True`` to make Scrapy abort HTTPS requests when the server certificate + is invalid or doesn't match the domain. + (:issue:`7379`) + +- The built-in HTTP :ref:`download handlers ` now + raise Scrapy-specific exceptions instead of implementation-specific ones, + to allow unified handling of similar problems caused by different + implementations. The default value of the :setting:`RETRY_EXCEPTIONS` + setting was updated replacing Twisted-specific exceptions with these new + ones. The exceptions: + + - :exc:`~scrapy.exceptions.CannotResolveHostError` + + - :exc:`~scrapy.exceptions.DownloadCancelledError` + + - :exc:`~scrapy.exceptions.DownloadConnectionRefusedError` + + - :exc:`~scrapy.exceptions.DownloadFailedError` + + - :exc:`~scrapy.exceptions.DownloadTimeoutError` + + - :exc:`~scrapy.exceptions.ResponseDataLossError` + + - :exc:`~scrapy.exceptions.UnsupportedURLSchemeError` + + (:issue:`7208`) + +- Added the :signal:`memusage_warning_reached` signal emitted by the + :class:`~scrapy.extensions.memusage.MemoryUsage` extension when the memory + usage reaches :setting:`MEMUSAGE_WARNING_MB`. + (:issue:`7249`, :issue:`7263`) + +- Added + :meth:`Headers.to_tuple_list() ` + that returns headers as a list of ``(key, value)`` tuples. + (:issue:`7239`) + +- :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler` now uses the + download handler configured for the ``"https"`` scheme to make requests + instead of always using + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + (:issue:`7369`, :issue:`7370`) + +- Added :func:`scrapy.utils.misc.walk_modules_iter` as a replacement for + ``scrapy.utils.misc.walk_modules()`` that returns an iterable instead of a + list. + (:issue:`7388`) + +Improvements +~~~~~~~~~~~~ + +- :func:`asyncio.to_thread` is now used instead of + :func:`twisted.internet.threads.deferToThread` in the built-in feed + storages, media pipeline storages and the + :func:`scrapy.utils.decorators.inthread` decorator when available. + (:issue:`7183`, :issue:`7184`, :issue:`7349`) + +- Improved memory footprint of :class:`~scrapy.Request` and + :class:`~scrapy.http.Response` objects by adding ``__slots__`` and omitting + empty lists and dicts in some internal attributes. + (:issue:`7036`, :issue:`7367`, :issue:`7374`) + +- :class:`~scrapy.core.downloader.contextfactory._ScrapyClientContextFactory` + no longer mutates the SSL context, to avoid the behavior that was + deprecated in pyOpenSSL 25.1.0. + (:issue:`6859`, :issue:`7353`) + +- Improved memory usage of :class:`~scrapy.spiders.sitemap.SitemapSpider` and + :class:`scrapy.utils.sitemap.Sitemap`. + (:issue:`3529`, :issue:`7007`) + +- Improved the scheduling behavior of + :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` when crawling + multiple domains. + (:issue:`7293`, :issue:`7351`) + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now handle + TLS verbose logging (see :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING`) + directly instead of relying on + :class:`~scrapy.core.downloader.contextfactory._ScrapyClientContextFactory`. + (:issue:`7387`) + +- The server certificate verification code now correctly handles certificates + with IP addresses in ``subjectAltName``. + (:issue:`7353`) + +- Improved reliability of :func:`scrapy.utils.trackref.get_oldest`. + (:issue:`1758`, :issue:`7375`) + +- Other code refactoring and improvements. + (:issue:`7210`, :issue:`7238`, :issue:`7376`, :issue:`7386`, :issue:`7395`, + :issue:`7405`, :issue:`7410`) + +Bug fixes +~~~~~~~~~ + +- :ref:`Media pipelines ` should now wait for uploads + to asynchronous storages (e.g. + :class:`~scrapy.pipelines.files.S3FilesStore`) to complete. + (:issue:`2183`, :issue:`6369`, :issue:`7182`) + +- Fixed merging ``*_BASE`` settings (e.g. merging + :setting:`DOWNLOADER_MIDDLEWARES` with + :setting:`DOWNLOADER_MIDDLEWARES_BASE`) when a component is referred to by + a class object in one setting and by a string import path in the other one. + (:issue:`6912`, :issue:`6993`) + +- ``scrapy runspider`` and ``scrapy crawl`` now set the exit code to 1 if an + exception happened early (this was broken since Scrapy 2.13.0). + (:issue:`6820`, :issue:`7255`) + +- Fixed repeated warnings about data loss (see + :setting:`DOWNLOAD_FAIL_ON_DATALOSS`) not being suppressed in + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + (:issue:`7222`) + +- Improved FTP connection management in + :class:`scrapy.pipelines.files.FTPFilesStore`. + (:issue:`7256`) + +- Fixed the ``spider`` variable in the :ref:`shell `, which + wasn't available since Scrapy 2.13.0. + (:issue:`7395`) + +Documentation +~~~~~~~~~~~~~ + +- The ``llms.txt`` and ``llms-full.txt`` files and Markdown versions of pages + are now generated when the HTML documentation is built. + (:issue:`7380`) + +- Added a "Copy as Markdown" button to the HTML documentation. + (:issue:`7380`) + +- Added :ref:`docs for using Pydantic models as items `. + (:issue:`6955`, :issue:`6966`) + +- Documented :ref:`job directory contents `. + (:issue:`4842`, :issue:`5260`) + +- Improved docs for :attr:`~scrapy.Request.dont_filter`. + (:issue:`6398`, :issue:`7245`) + +- Clarified that settings related to :setting:`TWISTED_DNS_RESOLVER` are only + taken into account if the selected resolver supports them. + (:issue:`7385`) + +- Other documentation improvements and fixes. + (:issue:`7248`, :issue:`7274`, :issue:`7406`, :issue:`7408`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added the ``no-reactor`` test environment that doesn't install a Twisted + reactor and uses ``pytest-asyncio`` instead of ``pytest-twisted`` to run + asynchronous test functions. + (:issue:`6952`, :issue:`7189`, :issue:`7233`, :issue:`7234`, :issue:`7254`, + :issue:`7259`) + +- Fixed running tests with ``pytest-xdist``. + (:issue:`7216`, :issue:`7257`) + +- Type hints improvements and fixes. + (:issue:`7300`, :issue:`7331`) + +- CI and test improvements and fixes. + (:issue:`7060`, + :issue:`7223`, + :issue:`7232`, + :issue:`7241`, + :issue:`7250`, + :issue:`7256`, + :issue:`7276`, + :issue:`7277`, + :issue:`7279`, + :issue:`7329`, + :issue:`7363`, + :issue:`7381`, + :issue:`7402`) + +.. _release-2.14.2: + +Scrapy 2.14.2 (2026-03-12) +-------------------------- + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Values from the ``Referrer-Policy`` header of HTTP responses are no longer + executed as Python callables. See the `cwxj-rr6w-m6w7`_ security advisory + for details. + + .. _cwxj-rr6w-m6w7: https://github.com/scrapy/scrapy/security/advisories/GHSA-cwxj-rr6w-m6w7 + +- In line with the `standard + `__, 301 redirects of + ``POST`` requests are converted into ``GET`` requests. + + Converting to a ``GET`` request implies not only a method change, but also + omitting the body and ``Content-*`` headers in the redirect request. On + cross-origin redirects (for example, cross-domain redirects), this is + effectively a security bug fix for scenarios where the body contains + secrets. + +Deprecations +~~~~~~~~~~~~ + +- Passing a response URL string as the first positional argument to + :meth:`scrapy.spidermiddlewares.referer.RefererMiddleware.policy` is + deprecated. Pass a :class:`~scrapy.http.Response` instead. + + The parameter has also been renamed to ``response`` to reflect this change. + The old parameter name (``resp_or_url``) is deprecated. + +New features +~~~~~~~~~~~~ + +- Added a new setting, :setting:`REFERRER_POLICIES`, to allow customizing + supported referrer policies. + +Bug fixes +~~~~~~~~~ + +- Made additional redirect scenarios convert to ``GET`` in line with the + `standard `__: + + - Only ``POST`` 302 redirects are converted into ``GET`` requests; other + methods are preserved. + + - ``HEAD`` 303 redirects are not converted into ``GET`` requests. + + - ``GET`` 303 redirects do not have their body or standard ``Content-*`` + headers removed. + +- Redirects where the original request body is dropped now also have their + ``Content-Encoding``, ``Content-Language`` and ``Content-Location`` headers + removed, in addition to the ``Content-Type`` and ``Content-Length`` headers + that were already being removed. + +- Redirects now preserve the source URL fragment if the redirect URL does not + include one. This is useful when using browser-based download handlers, + such as `scrapy-playwright`_ or `scrapy-zyte-api`_, while letting Scrapy + handle redirects. + + .. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright + .. _scrapy-zyte-api: https://scrapy-zyte-api.readthedocs.io/en/latest/ + +- The ``Referer`` header is now removed on redirect if + :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` is disabled. + +- The handling of the ``Referer`` header on redirects now takes into account + the ``Referer-Policy`` header of the response that triggers the redirect. + +.. _release-2.14.1: + +Scrapy 2.14.1 (2026-01-12) +-------------------------- + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.utils.defer.maybeDeferred_coro()`` is deprecated. (:issue:`7212`) + +Bug fixes +~~~~~~~~~ + +- Fixed custom stats collectors that require a ``spider`` argument in their + ``open_spider()`` and ``close_spider()`` methods not receiving the + argument when called by the engine. + + Note, however, that the ``spider`` argument is now deprecated and will stop + being passed in a future version of Scrapy. + + (:issue:`7213`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Replaced deprecated ``codecov/test-results-action@v1`` GitHub Action with + ``codecov/codecov-action@v5``. + (:issue:`7180`, :issue:`7215`) + +.. _release-2.14.0: + +Scrapy 2.14.0 (2026-01-05) +-------------------------- + +Highlights: + +- More coroutine-based replacements for Deferred-based APIs + +- The default priority queue is now ``DownloaderAwarePriorityQueue`` + +- Dropped support for Python 3.9 and PyPy 3.10 + +- Improved and documented the API for custom download handlers + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for Python 3.9. + (:issue:`7121`) + +- Dropped support for PyPy 3.10. + (:issue:`7050`) + +- Increased the minimum versions of the following dependencies: + + - lxml_: 4.6.0 → 4.6.4 + + - Pillow_ (optional dependency): 8.0.0 → 8.3.2 + + - botocore_ (optional dependency): 1.4.87 → 1.13.45 + +- Restored support for ``brotlicffi`` dropped in Scrapy 2.13.4. Its minimum + supported version is now ``1.2.0.0``. + (:issue:`7160`) Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -14,6 +1144,7 @@ Backward-incompatible changes may now need to set the :setting:`FORCE_CRAWLER_PROCESS` setting to ``True`` when running Scrapy via :ref:`its command-line tool ` to avoid a reactor mismatch exception. + (:issue:`6845`) - The ``log_count/*`` stats no longer count some of the early messages that they counted before. While the earliest log messages, emitted before the @@ -52,6 +1183,17 @@ Backward-incompatible changes - :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer` + (:issue:`6930`) + +- Scrapy no longer passes a ``spider`` argument to any methods of the + :setting:`stats collector `. It wasn't passed in many of the + calls even in older Scrapy versions, so we don't expect existing custom + stats collector implementations to require a ``spider`` argument. If your + implementation needs a :class:`~scrapy.Spider` instance, you can get it + from the :class:`~scrapy.crawler.Crawler` instance passed to the + constructor. + (:issue:`7011`) + - :class:`scrapy.middleware.MiddlewareManager` no longer includes code for handling ``open_spider()`` and ``close_spider()`` component methods. As this code was only used for pipelines it was moved into @@ -66,6 +1208,8 @@ Backward-incompatible changes - Code in ``scrapy.middleware.MiddlewareManager._add_middleware()`` that processes ``open_spider()`` and ``close_spider()`` component methods. + (:issue:`7006`) + - :meth:`scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware.process_request` now returns a coroutine, previously it returned a :class:`~twisted.internet.defer.Deferred` object or ``None``. The @@ -73,6 +1217,561 @@ Backward-incompatible changes change only impacts code that subclasses :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` or calls its methods directly. + (:issue:`6802`) + +- The built-in :ref:`download handlers ` have been + refactored, changing the signatures of their methods. This change should + only affect user code that subclasses any of these handlers or calls their + methods directly. + (:issue:`6778`, :issue:`7164`) + +- :meth:`scrapy.pipelines.media.MediaPipeline.process_item` now returns a + coroutine, previously it returned a + :class:`~twisted.internet.defer.Deferred` object. This + change only impacts code that calls this method directly. + (:issue:`7177`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The ``from_settings()`` method of the following components, deprecated in + Scrapy 2.12.0, is removed. You should use ``from_crawler()`` instead. + + - :class:`scrapy.dupefilters.RFPDupeFilter` + - :class:`scrapy.mail.MailSender` + - :class:`scrapy.middleware.MiddlewareManager` + - :class:`scrapy.core.downloader.contextfactory.ScrapyClientContextFactory` + - :class:`scrapy.pipelines.files.FilesPipeline` + - :class:`scrapy.pipelines.images.ImagesPipeline` + + (:issue:`7126`) + +- Scrapy no longer calls ``from_settings()`` methods of 3rd-party + :ref:`components `, deprecated in Scrapy 2.12.0. You + should define a ``from_crawler()`` method instead. + (:issue:`7126`) + +- The initialization flow of :class:`scrapy.pipelines.media.MediaPipeline` + and its subclasses was simplified, it now mandates ``from_crawler()`` + methods and ``crawler`` arguments of ``__init__()`` methods. Not using + these was deprecated in Scrapy 2.12.0. + (:issue:`7126`) + +- The ``REQUEST_FINGERPRINTER_IMPLEMENTATION`` setting, deprecated in Scrapy + 2.12.0, is removed. + (:issue:`7126`) + +- The ``scrapy.utils.misc.create_instance()`` function, deprecated in Scrapy + 2.12.0, is removed. Use :func:`scrapy.utils.misc.build_from_crawler` + instead. + (:issue:`7126`) + +- The ``scrapy.core.downloader.Downloader._get_slot_key()`` function, + deprecated in Scrapy 2.12.0, is removed. Use + :meth:`scrapy.core.downloader.Downloader.get_slot_key` instead. + (:issue:`7126`) + +- The ``scrapy.twisted_version`` attribute, deprecated in Scrapy 2.12.0, is + removed. You should instead use the :attr:`twisted.version` attribute + directly. + (:issue:`7126`) + +- The following utility functions, deprecated in Scrapy 2.12.0, are removed: + + - ``scrapy.utils.defer.process_chain_both()`` + - ``scrapy.utils.python.equal_attributes()`` + - ``scrapy.utils.python.flatten()`` + - ``scrapy.utils.python.iflatten()`` + - ``scrapy.utils.request.request_authenticate()`` + - ``scrapy.utils.test.assert_samelines()`` + + (:issue:`7126`) + +- ``scrapy.utils.serialize.ScrapyJSONDecoder``, deprecated in Scrapy 2.12.0, + is removed. + (:issue:`7126`) + +- The ``scrapy.extensions.feedexport.build_storage()`` function, deprecated + in Scrapy 2.12.0, is removed, you can instead call the builder callable + directly. + (:issue:`7126`) + +- ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware``, deprecated in + Scrapy 2.11.2, is removed. + :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` should be + used instead. + (:issue:`6926`) + +Deprecations +~~~~~~~~~~~~ + +- The following methods that return a + :class:`~twisted.internet.defer.Deferred` are deprecated in favor of their + coroutine-based replacements: + + - :class:`scrapy.core.downloader.handlers.DownloadHandlers` + + - ``download_request()`` (use + :meth:`~scrapy.core.downloader.handlers.DownloadHandlers.download_request_async`) + + - :class:`scrapy.core.downloader.middleware.DownloaderMiddlewareManager` + + - ``download()`` (use + :meth:`~scrapy.core.downloader.middleware.DownloaderMiddlewareManager.download_async`) + + - :class:`scrapy.core.engine.ExecutionEngine` + + - ``start()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.start_async`) + + - ``stop()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.stop_async`) + + - ``close()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.close_async`) + + - ``open_spider()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.open_spider_async`) + + - ``close_spider()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.close_spider_async`) + + - ``download()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.download_async`) + + - :class:`scrapy.core.scraper.Scraper` + + - ``open_spider()`` (use + :meth:`~scrapy.core.scraper.Scraper.open_spider_async`) + + - ``call_spider()`` (use + :meth:`~scrapy.core.scraper.Scraper.call_spider_async`) + + - ``close_spider()`` (use + :meth:`~scrapy.core.scraper.Scraper.close_spider_async`) + + - ``handle_spider_output()`` (use + :meth:`~scrapy.core.scraper.Scraper.handle_spider_output_async`) + + - ``start_itemproc()`` (use + :meth:`~scrapy.core.scraper.Scraper.start_itemproc_async`) + + - :class:`scrapy.core.spidermw.SpiderMiddlewareManager` + + - ``scrape_response()`` (use + :meth:`~scrapy.core.spidermw.SpiderMiddlewareManager.scrape_response_async`) + + - :class:`scrapy.crawler.Crawler` + + - ``stop()`` (use :meth:`~scrapy.crawler.Crawler.stop_async`) + + - :class:`scrapy.pipelines.ItemPipelineManager` + + - ``process_item()`` (use + :meth:`~scrapy.pipelines.ItemPipelineManager.process_item_async`) + + - ``open_spider()`` (use + :meth:`~scrapy.pipelines.ItemPipelineManager.open_spider_async`) + + - ``close_spider()`` (use + :meth:`~scrapy.pipelines.ItemPipelineManager.close_spider_async`) + + - :class:`scrapy.signalmanager.SignalManager` + + - ``send_catch_log_deferred()`` (use + :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_async`) + + - ``scrapy.utils.signal.send_catch_log_deferred()`` (use + :func:`scrapy.utils.signal.send_catch_log_async`) + + (:issue:`6791`, :issue:`6842`, :issue:`6979`, :issue:`6997`, :issue:`6999`, + :issue:`7005`, :issue:`7043`, :issue:`7069`, :issue:`7161`, :issue:`7164`) + +- The following spider attributes are deprecated in favor of settings: + + - ``download_maxsize`` (use :setting:`DOWNLOAD_MAXSIZE`) + + - ``download_timeout`` (use :setting:`DOWNLOAD_TIMEOUT`) + + - ``download_warnsize`` (use :setting:`DOWNLOAD_WARNSIZE`) + + - ``max_concurrent_requests`` (use :setting:`CONCURRENT_REQUESTS`) + + - ``user_agent`` (use :setting:`USER_AGENT`) + + (:issue:`6988`, :issue:`6994`, :issue:`7038`, :issue:`7039`, :issue:`7117`, + :issue:`7176`) + +- Returning a :class:`~twisted.internet.defer.Deferred` from the following + user-defined functions is deprecated in favor of defining them as coroutine + functions: + + - spider callbacks and errbacks (which was never officially supported and + may work incorrectly) + + - the ``process_request()``, ``process_response()`` and + ``process_exception()`` methods of custom downloader middlewares + + - the ``process_item()``, ``open_spider()`` and ``close_spider()`` methods + of custom pipelines + + - signal handlers + + - the ``download_request()`` and ``close()`` methods of custom download + handlers + + (:issue:`6718`, :issue:`6778`, :issue:`7069`, :issue:`7147`, :issue:`7148`, + :issue:`7149`, :issue:`7150`, :issue:`7151`, :issue:`7161`, :issue:`7164`, + :issue:`7179`) + +- Passing a ``spider`` argument to the following methods is deprecated: + + - :meth:`scrapy.core.spidermw.SpiderMiddlewareManager.process_start` + + - :meth:`scrapy.core.downloader.Downloader.fetch` + + - :meth:`scrapy.core.downloader.Downloader._get_slot` + + - :meth:`scrapy.core.downloader.handlers.DownloadHandlers.download_request` + + - all public methods of :class:`scrapy.statscollectors.StatsCollector` + + - :meth:`scrapy.spidermiddlewares.base.BaseSpiderMiddleware.process_spider_output` + + - :meth:`scrapy.spidermiddlewares.base.BaseSpiderMiddleware.process_spider_output_async` + + - all ``process_*()`` methods of built-in downloader middlewares + + - all ``process_*()`` methods of built-in spider middlewares + + - :meth:`scrapy.pipelines.media.MediaPipeline.open_spider` + + - :meth:`scrapy.pipelines.media.MediaPipeline.process_item` + + (:issue:`6750`, :issue:`6927`, :issue:`6984`, :issue:`7006`, :issue:`7011`, + :issue:`7033`, :issue:`7037`, :issue:`7045`, :issue:`7178`) + +- Instantiating subclasses of :class:`scrapy.middleware.MiddlewareManager` + without a :class:`~scrapy.crawler.Crawler` instance is deprecated. + (:issue:`6984`) + +- For the following user-defined functions and methods requiring a ``spider`` + argument is deprecated, if you need a :class:`~scrapy.Spider` instance + inside them you should get it from the :class:`~scrapy.crawler.Crawler` + instance (you may need to refactor your code to save that instance in e.g. + the ``from_crawler()`` method): + + - the ``process_request()``, ``process_response()`` and + ``process_exception()`` methods of custom downloader middlewares + + - the ``process_spider_input()``, ``process_spider_output()``, + ``process_spider_output_async()`` and ``process_spider_exception()`` + methods of custom spider middlewares + + - the ``process_item()`` method of custom pipelines + + - the ``fetch()`` method of a custom :setting:`DOWNLOADER` + + (:issue:`6927`, :issue:`6984`, :issue:`7006`, :issue:`7037`) + +- The following things in custom download handlers are deprecated: + + - not having a ``lazy`` attribute (you should define it as ``True`` if you + want to keep the current behavior) + + - returning a :class:`~twisted.internet.defer.Deferred` from the + ``download_request()`` method (you should refactor it to return a + coroutine; you also need to remove the ``spider`` argument when doing + this) + + - not having a ``close()`` method, having a synchronous one or one that + returns a :class:`~twisted.internet.defer.Deferred` (you should refactor + it to return a coroutine or add an empty one if you don't have it) + + (:issue:`6778`, :issue:`7164`) + +- Custom implementations of :setting:`ITEM_PROCESSOR` should now define + ``process_item_async()``, ``open_spider_async()`` and + ``close_spider_async()`` methods instead of, or in addition to, + ``process_item()``, ``open_spider()`` and ``close_spider()``. + (:issue:`7005`, :issue:`7043`) + +- The ``CONCURRENT_REQUESTS_PER_IP`` setting is deprecated, use + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` instead. + (:issue:`6917`, :issue:`6921`) + +- The ``scrapy.core.downloader.handlers.http`` module is deprecated. You + should import + :class:`scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + directly instead of importing the + ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler`` alias. + (:issue:`7079`) + +- The ``scrapy.utils.decorators.defers()`` decorator is deprecated, you can + use :func:`twisted.internet.defer.maybeDeferred` directly or reimplement + this decorator in your code. + (:issue:`7164`) + +- ``scrapy.spiders.CrawlSpider._parse_response()`` is deprecated, use + :meth:`scrapy.spiders.CrawlSpider.parse_with_rules` instead. + (:issue:`4463`, :issue:`6804`) + +- The functions that add a delay to a Deferred are deprecated, their + underlying Twisted functions can be used instead, either directly if a + delay isn't needed, or with some explicit way to add a delay if it's + needed: + + - ``scrapy.utils.defer.mustbe_deferred()`` (you can use + :func:`twisted.internet.defer.maybeDeferred`) + + - ``scrapy.utils.defer.defer_succeed()`` (you can use + :func:`twisted.internet.defer.succeed`) + + - ``scrapy.utils.defer.defer_fail()`` (you can use + :func:`twisted.internet.defer.fail`) + + - ``scrapy.utils.defer.defer_result()`` (you can use + :func:`twisted.internet.defer.succeed` and + :func:`twisted.internet.defer.fail`) + + (:issue:`6937`) + +New features +~~~~~~~~~~~~ + +- Added :class:`scrapy.crawler.AsyncCrawlerProcess` and + :class:`scrapy.crawler.AsyncCrawlerRunner` as counterparts to + :class:`~scrapy.crawler.CrawlerProcess` and + :class:`~scrapy.crawler.CrawlerRunner` that offer coroutine-based APIs. + (:issue:`6789`, :issue:`6790`, :issue:`6796`, :issue:`6817`, :issue:`6845`, + :issue:`7034`) + +- Added coroutine counterparts to some of the Deferred-based APIs: + + - :class:`scrapy.core.downloader.handlers.DownloadHandlers` + + - :meth:`~scrapy.core.downloader.handlers.DownloadHandlers.download_request_async` + (to ``download_request()``) + + - :class:`scrapy.core.downloader.middleware.DownloaderMiddlewareManager` + + - :meth:`~scrapy.core.downloader.middleware.DownloaderMiddlewareManager.download_async` + (to ``download()``) + + - :class:`scrapy.core.engine.ExecutionEngine` + + - :meth:`~scrapy.core.engine.ExecutionEngine.start_async` (to + ``start()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.stop_async` (to + ``stop()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.close_async` (to + ``close()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.open_spider_async` (to + ``open_spider()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.close_spider_async` (to + ``close_spider()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.download_async` (to + ``download()``) + + - :class:`scrapy.core.scraper.Scraper` + + - :meth:`~scrapy.core.scraper.Scraper.open_spider_async` (to + ``open_spider()``) + + - :meth:`~scrapy.core.scraper.Scraper.close_spider_async` (to + ``close_spider()``) + + - :meth:`~scrapy.core.scraper.Scraper.start_itemproc_async` (to + ``start_itemproc()``) + + - :class:`scrapy.crawler.Crawler` + + - :meth:`~scrapy.crawler.Crawler.crawl_async` (to ``crawl()``) + + - :meth:`~scrapy.crawler.Crawler.stop_async` (to ``stop()``) + + - :class:`scrapy.pipelines.ItemPipelineManager` + + - :meth:`~scrapy.pipelines.ItemPipelineManager.process_item_async` (to + ``process_item()``) + + - :meth:`~scrapy.pipelines.ItemPipelineManager.open_spider_async` (to + ``open_spider()``) + + - :meth:`~scrapy.pipelines.ItemPipelineManager.close_spider_async` (to + ``close_spider()``) + + - :class:`scrapy.signalmanager.SignalManager` + + - :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_async` (to + ``send_catch_log_deferred()``) + + (:issue:`6781`, :issue:`6791`, :issue:`6792`, :issue:`6795`, :issue:`6801`, + :issue:`6817`, :issue:`6842`, :issue:`6997`, :issue:`7005`, :issue:`7043`, + :issue:`7069`,:issue:`7164`, :issue:`7202`) + +- The default value of the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is now + ``'scrapy.pqueues.DownloaderAwarePriorityQueue'``. + (:issue:`6924`, :issue:`6940`) + +- Added :class:`scrapy.extensions.logcount.LogCount`, an enabled-by-default + extension that is responsible for the ``log_count/*`` stats. Previously, + this code was in :class:`scrapy.crawler.Crawler` and couldn't be disabled. + (:issue:`7046`) + +- Added :meth:`scrapy.spiders.CrawlSpider.parse_with_rules` as a public + replacement for ``_parse_response()``. + (:issue:`4463`, :issue:`6804`) + +- Added :func:`scrapy.utils.asyncio.is_asyncio_available` as an alternative + to :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` with a + future-proof name and semantics. + (:issue:`6827`) + +- The API for :ref:`download handlers `, previously + undocumented, has been modernized and documented. An optional base class, + :class:`scrapy.core.downloader.handlers.base.BaseDownloadHandler`, has been + added to simplify writing custom download handlers that conform to the + current API. + (:issue:`4944`, :issue:`6778`, :issue:`7164`) + +- Added :func:`scrapy.utils.defer.ensure_awaitable`, which can be helpful to + call user-defined functions that can return coroutines, Deferreds or + values directly. + (:issue:`7005`) + +- The ``requests.seen`` file, written by + :class:`~scrapy.dupefilters.RFPDupeFilter` when :ref:`job persistence + ` is enabled, now uses line buffering to reduce data loss in + spider crashes. + (:issue:`6019`, :issue:`7094`) + +- Images downloaded by :class:`~scrapy.pipelines.images.ImagesPipeline` are + now automatically transposed based on EXIF data. + (:issue:`6525`, :issue:`6975`) + +Improvements +~~~~~~~~~~~~ + +- Refactored internal functions to use coroutines instead of Deferreds. + (:issue:`6795`, :issue:`6852`, :issue:`6855`, :issue:`6858`, :issue:`7159`) + +- Commands that don't need a :class:`~scrapy.crawler.CrawlerProcess` instance + no longer create it. + (:issue:`6824`) + +- Improved :command:`shell` help formatting when using IPython 9+. + (:issue:`6915`, :issue:`6980`) + +Bug fixes +~~~~~~~~~ + +- Setting :setting:`FILES_STORE` or :setting:`IMAGES_STORE` to ``None`` now + correctly disables the respective pipeline. + (:issue:`6964`, :issue:`6969`) + +- :class:`~scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware` now + uses the URL set in the ```` tag as the base URL when redirecting to + a relative URL. + (:issue:`7042`, :issue:`7047`) + +- Passing ``None`` as a value of the :reqmeta:`download_slot` request meta + key is now handled in the same way as not setting this meta key at all. + (:issue:`7172`) + +- Fixed parsing of the first line of ``robots.txt`` files that have a BOM. + (:issue:`6195`, :issue:`7095`) + +Documentation +~~~~~~~~~~~~~ + +- Added :ref:`documentation ` about download + handlers, their API and built-in handlers. + (:issue:`4944`, :issue:`7164`) + +- Added a section about the `scrapy-spider-metadata`_ library to the + :ref:`spider argument docs `. + (:issue:`6676`, :issue:`6957`, :issue:`7116`) + + .. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/ + +- Improved :ref:`the docs ` about coroutine-based + and Deferred-based APIs. + (:issue:`6800`, :issue:`7146`) + +- Other documentation improvements and fixes. + (:issue:`7058`, :issue:`7076`, :issue:`7109`, :issue:`7195`, :issue:`7198`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Switched from ``twisted.trial`` to ``pytest-twisted`` and replaced + remaining ``unittest`` and ``twisted.trial`` features with ``pytest`` ones. + (:issue:`6658`, :issue:`6873`, :issue:`6884`, :issue:`6938`) + +- Enabled fancy ``pytest`` asserts. + (:issue:`6888`) + +- Added `Sphinx Lint`_ to the ``pre-commit`` configuration. + (:issue:`6920`) + + .. _Sphinx Lint: https://github.com/sphinx-contrib/sphinx-lint + +- CI and test improvements and fixes. + (:issue:`6649`, + :issue:`6769`, + :issue:`6821`, + :issue:`6835`, + :issue:`6836`, + :issue:`6846`, + :issue:`6883`, + :issue:`6885`, + :issue:`6889`, + :issue:`6905`, + :issue:`6928`, + :issue:`6933`, + :issue:`6941`, + :issue:`6942`, + :issue:`6945`, + :issue:`6947`, + :issue:`6960`, + :issue:`6968`, + :issue:`6972`, + :issue:`6974`, + :issue:`6996`, + :issue:`7003`, + :issue:`7012`, + :issue:`7013`, + :issue:`7050`, + :issue:`7059`, + :issue:`7070`, + :issue:`7073`, + :issue:`7118`, + :issue:`7127`, + :issue:`7141`, + :issue:`7143`, + :issue:`7145`, + :issue:`7173`) + +- Code cleanups. + (:issue:`6803`, + :issue:`6838`, + :issue:`6849`, + :issue:`6875`, + :issue:`6876`, + :issue:`6892`, + :issue:`6930`, + :issue:`6949`, + :issue:`6970`, + :issue:`6977`, + :issue:`6986`, + :issue:`7008`, + :issue:`7177`) .. _release-2.13.4: @@ -254,12 +1953,12 @@ Highlights: - The asyncio reactor is now enabled by default - Replaced ``start_requests()`` (sync) with :meth:`~scrapy.Spider.start` - (async) and changed how it is iterated. + (async) and changed how it is iterated - Added the :reqmeta:`allow_offsite` request meta key -- :ref:`Spider middlewares that don't support asynchronous spider output - ` are deprecated +- Spider middlewares that don't support asynchronous spider output are + deprecated - Added a base class for :ref:`universal spider middlewares ` @@ -397,13 +2096,11 @@ Deprecations ``start_queue_cls`` parameter. (:issue:`6752`) -- :ref:`Spider middlewares that don't support asynchronous spider output - ` are deprecated. The async iterable - downgrading feature, needed for using such middlewares with asynchronous - callbacks and with other spider middlewares that produce asynchronous - iterables, is also deprecated. Please update all such middlewares to - support asynchronous spider output. - (:issue:`6664`) +- Spider middlewares that don't support asynchronous spider output are + deprecated. The async iterable downgrading feature, needed for using such + middlewares with asynchronous callbacks and with other spider middlewares + that produce asynchronous iterables, is also deprecated. Please update all + such middlewares to support asynchronous spider output. (:issue:`6664`) - Functions that were imported from :mod:`w3lib.url` and re-exported in :mod:`scrapy.utils.url` are now deprecated, you should import them from @@ -455,9 +2152,9 @@ Deprecations - The following modules and functions used only in tests are deprecated: - - the ``scrapy/utils/testproc`` module + - the ``scrapy.utils.testproc`` module - - the ``scrapy/utils/testsite`` module + - the ``scrapy.utils.testsite`` module - ``scrapy.utils.test.assert_gcs_environ()`` @@ -662,9 +2359,8 @@ Documentation - Documented the setting values set in the default project template. (:issue:`6762`, :issue:`6775`) -- Improved the :ref:`docs ` about asynchronous - iterable support in spider middlewares. - (:issue:`6688`) +- Improved the docs about asynchronous iterable support in spider + middlewares. (:issue:`6688`) - Improved the :ref:`docs ` about using :class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based @@ -828,7 +2524,7 @@ Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - User-defined cookies for HTTPS requests will have the ``secure`` flag set - to ``True`` unless it's set to ``False`` explictly. This is important when + to ``True`` unless it's set to ``False`` explicitly. This is important when these cookies are reused in HTTP requests, e.g. after a redirect to an HTTP URL. (:issue:`6357`) @@ -863,7 +2559,7 @@ Backward-incompatible changes ``crawler.settings`` instead. When they call ``__init__()`` of the base class they should pass the ``crawler`` argument to it too. - A ``from_settings()`` method shouldn't be defined. Class-specific - initialization code should go into either an overriden ``from_crawler()`` + initialization code should go into either an overridden ``from_crawler()`` method or into ``__init__()``. - It's now possible to override ``from_crawler()`` and it's not necessary to call ``MediaPipeline.from_crawler()`` in it if other recommendations @@ -2366,7 +4062,7 @@ Bug fixes that does not match the asyncio event loop actually installed (:issue:`5529`). -- Fixed :meth:`Headers.getlist ` +- Fixed :meth:`Headers.getlist() ` returning only the last header (:issue:`5515`, :issue:`5526`). - Fixed :class:`LinkExtractor @@ -2611,7 +4307,7 @@ Modified requirements ~~~~~~~~~~~~~~~~~~~~~ - The h2_ dependency is now optional, only needed to - :ref:`enable HTTP/2 support `. (:issue:`5113`) + :ref:`enable HTTP/2 support `. (:issue:`5113`) .. _h2: https://pypi.org/project/h2/ @@ -2993,7 +4689,7 @@ Highlights: - Official Python 3.9 support -- Experimental :ref:`HTTP/2 support ` +- Experimental :ref:`HTTP/2 support ` - New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function to retry requests from spider callbacks @@ -3024,7 +4720,7 @@ Deprecations New features ~~~~~~~~~~~~ -- Experimental :ref:`HTTP/2 support ` through a new download handler +- Experimental :ref:`HTTP/2 support ` through a new download handler that can be assigned to the ``https`` protocol in the :setting:`DOWNLOAD_HANDLERS` setting. (:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`) @@ -3998,7 +5694,7 @@ Highlights: * :ref:`FTP support ` for media pipelines * New :attr:`Response.certificate ` attribute -* IPv6 support through :setting:`DNS_RESOLVER` +* IPv6 support through ``DNS_RESOLVER`` Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -4106,7 +5802,7 @@ New features :class:`twisted.internet.ssl.Certificate` object for HTTPS responses (:issue:`2726`, :issue:`4054`) -* A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support +* A new ``DNS_RESOLVER`` setting allows enabling IPv6 support (:issue:`1031`, :issue:`4227`) * A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring @@ -4186,7 +5882,7 @@ New features components already supported (:issue:`4126`) * :class:`scrapy.utils.python.MutableChain.__iter__` now returns ``self``, - `allowing it to be used as a sequence `_ + allowing it to be used as a sequence. (:issue:`4153`) @@ -4350,7 +6046,7 @@ The following changes may impact custom priority queue classes: * A new keyword parameter has been added: ``key``. It is a string that is always an empty string for memory queues and indicates the - :setting:`JOB_DIR` value for disk queues. + :setting:`JOBDIR` value for disk queues. * The parameter for disk queues that contains data from the previous crawl, ``startprios`` or ``slot_startprios``, is now passed as a @@ -4636,7 +6332,7 @@ Backward-incompatible changes consistency with similar classes (:issue:`3929`, :issue:`3982`) * If you are using a custom context factory - (:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`), its ``__init__`` method must + (``DOWNLOADER_CLIENTCONTEXTFACTORY``), its ``__init__`` method must accept two new parameters: ``tls_verbose_logging`` and ``tls_ciphers`` (:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`) @@ -4871,8 +6567,8 @@ Backward-incompatible changes ``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly. * :class:`~scrapy.crawler.Crawler`, - :class:`CrawlerRunner.crawl ` and - :class:`CrawlerRunner.create_crawler ` + :meth:`CrawlerRunner.crawl ` and + :meth:`CrawlerRunner.create_crawler ` no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they only accept a :class:`~scrapy.spiders.Spider` subclass now. @@ -4904,7 +6600,7 @@ New features ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be :ref:`enabled ` for a significant scheduling improvement on crawls targeting multiple web domains, at the - cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) + cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`) * A new :attr:`.Request.cb_kwargs` attribute provides a cleaner way to pass keyword arguments to callback methods @@ -6092,7 +7788,7 @@ This 1.1 release brings a lot of interesting features and bug fixes: selectors engine without needing to upgrade Scrapy. - HTTPS downloader now does TLS protocol negotiation by default, instead of forcing TLS 1.0. You can also set the SSL/TLS method - using the new :setting:`DOWNLOADER_CLIENT_TLS_METHOD`. + using the new ``DOWNLOADER_CLIENT_TLS_METHOD`` setting. - These bug fixes may require your attention: @@ -6127,8 +7823,7 @@ Keep reading for more details on other improvements and bug fixes. Beta Python 3 Support ~~~~~~~~~~~~~~~~~~~~~ -We have been `hard at work to make Scrapy run on Python 3 -`_. As a result, now +We have been hard at work to make Scrapy run on Python 3. As a result, now you can run spiders on Python 3.3, 3.4 and 3.5 (Twisted >= 15.5 required). Some features are still missing (and some may never be ported). @@ -6196,7 +7891,7 @@ Additional New Features and Enhancements - Other refactoring, optimizations and cleanup (:issue:`1476`, :issue:`1481`, :issue:`1477`, :issue:`1315`, :issue:`1290`, :issue:`1750`, :issue:`1881`). -.. _`Code of Conduct`: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md +.. _Code of Conduct: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md Deprecations and Removals @@ -6958,7 +8653,7 @@ Enhancements - Make ``RFPDupeFilter`` class easily subclassable (:issue:`533`) - Improve test coverage and forthcoming Python 3 support (:issue:`525`) - Promote startup info on settings and middleware to INFO level (:issue:`520`) -- Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`) +- Support partials in ``get_func_args`` util (:issue:`506`, :issue:`504`) - Allow running individual tests via tox (:issue:`503`) - Update extensions ignored by link extractors (:issue:`498`) - Add middleware methods to get files/images/thumbs paths (:issue:`490`) @@ -7384,7 +9079,7 @@ New features and settings - In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`) - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: - - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP` + - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP`` - check the documentation for more details - Added builtin caching DNS resolver (:rev:`2728`) - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) @@ -7526,7 +9221,7 @@ API changes - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` - ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler`` - Moved module: ``scrapy.contrib.spidermanager`` to ``scrapy.spidermanager`` -- Spider Manager singleton moved from ``scrapy.spider.spiders`` to the ``spiders` attribute of ``scrapy.project.crawler`` singleton. +- Spider Manager singleton moved from ``scrapy.spider.spiders`` to the ``spiders`` attribute of ``scrapy.project.crawler`` singleton. - moved Stats Collector classes: (#204) - ``scrapy.stats.collector.StatsCollector`` to ``scrapy.statscol.StatsCollector`` - ``scrapy.stats.collector.SimpledbStatsCollector`` to ``scrapy.contrib.statscol.SimpledbStatsCollector`` diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 000000000..a1f3a7468 --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,8 @@ +h2 +pydantic +scrapy-spider-metadata +sphinx +sphinx-notfound-page +sphinx-rtd-theme +sphinx-rtd-dark-mode +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 diff --git a/docs/requirements.txt b/docs/requirements.txt index 81a3119e4..a5cbad302 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,6 +1,197 @@ -pydantic==2.12.3 +# This file was autogenerated by uv via the following command: +# uv pip compile -p 3.13 docs/requirements.in -o docs/requirements.txt +alabaster==1.0.0 + # via sphinx +annotated-types==0.7.0 + # via pydantic +attrs==26.1.0 + # via + # service-identity + # twisted +automat==25.4.16 + # via twisted +babel==2.18.0 + # via sphinx +certifi==2026.2.25 + # via requests +cffi==2.0.0 + # via cryptography +charset-normalizer==3.4.6 + # via requests +constantly==23.10.4 + # via twisted +cryptography==46.0.6 + # via + # pyopenssl + # scrapy + # service-identity +cssselect==1.4.0 + # via + # parsel + # scrapy +defusedxml==0.7.1 + # via scrapy +docutils==0.22.4 + # via + # sphinx + # sphinx-markdown-builder + # sphinx-rtd-theme +filelock==3.25.2 + # via tldextract +h2==4.3.0 + # via -r docs/requirements.in +hpack==4.1.0 + # via h2 +hyperframe==6.1.0 + # via h2 +hyperlink==21.0.0 + # via twisted +idna==3.11 + # via + # hyperlink + # requests + # tldextract +imagesize==2.0.0 + # via sphinx +incremental==24.11.0 + # via twisted +itemadapter==0.13.1 + # via + # itemloaders + # scrapy +itemloaders==1.4.0 + # via scrapy +jinja2==3.1.6 + # via sphinx +jmespath==1.1.0 + # via + # itemloaders + # parsel +lxml==6.0.2 + # via + # parsel + # scrapy +markupsafe==3.0.3 + # via jinja2 +packaging==26.0 + # via + # incremental + # parsel + # scrapy + # scrapy-spider-metadata + # sphinx + # sphinx-scrapy +parsel==1.11.0 + # via + # itemloaders + # scrapy +protego==0.6.0 + # via scrapy +pyasn1==0.6.3 + # via + # pyasn1-modules + # service-identity +pyasn1-modules==0.4.2 + # via service-identity +pycparser==3.0 + # via cffi +pydantic==2.12.5 + # via + # -r docs/requirements.in + # scrapy-spider-metadata +pydantic-core==2.41.5 + # via pydantic +pydispatcher==2.0.7 + # via scrapy +pygments==2.19.2 + # via sphinx +pyopenssl==26.0.0 + # via scrapy +queuelib==1.9.0 + # via scrapy +requests==2.33.0 + # via + # requests-file + # sphinx + # tldextract +requests-file==3.0.1 + # via tldextract +roman-numerals==4.1.0 + # via sphinx +scrapy==2.14.2 + # via scrapy-spider-metadata scrapy-spider-metadata==0.2.0 -sphinx==8.1.3 -sphinx-notfound-page==1.0.4 -sphinx-rtd-theme==3.0.2 + # via -r docs/requirements.in +service-identity==24.2.0 + # via scrapy +snowballstemmer==3.0.1 + # via sphinx +sphinx==9.1.0 + # via + # -r docs/requirements.in + # sphinx-copybutton + # sphinx-last-updated-by-git + # sphinx-llms-txt + # sphinx-markdown-builder + # sphinx-notfound-page + # sphinx-rtd-theme + # sphinx-scrapy + # sphinxcontrib-jquery +sphinx-copybutton==0.5.2 + # via sphinx-scrapy +sphinx-last-updated-by-git==0.3.8 + # via sphinx-sitemap +sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69 + # via sphinx-scrapy +sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@cfe4c0bfd7b4542f7e6b65a58cdf9ec765829940 + # via sphinx-scrapy +sphinx-notfound-page==1.1.0 + # via -r docs/requirements.in sphinx-rtd-dark-mode==1.3.0 + # via -r docs/requirements.in +sphinx-rtd-theme==3.1.0 + # via + # -r docs/requirements.in + # sphinx-rtd-dark-mode +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737 + # via -r docs/requirements.in +sphinx-sitemap==2.9.0 + # via sphinx-scrapy +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +tabulate==0.10.0 + # via sphinx-markdown-builder +tldextract==5.3.1 + # via scrapy +twisted==25.5.0 + # via scrapy +typing-extensions==4.15.0 + # via + # pydantic + # pydantic-core + # twisted + # typing-inspection +typing-inspection==0.4.2 + # via pydantic +urllib3==2.6.3 + # via requests +w3lib==2.4.1 + # via + # parsel + # scrapy +zope-interface==8.2 + # via + # scrapy + # twisted diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index dba14e749..78b1ab92b 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -98,9 +98,9 @@ recommend that such custom components should be written in the following way: (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default setting to the component provided by the add-on (e.g. ``MyDownloadHandler``). If the fallback setting is already set by the user, - they shouldn't change it. + it should not be changed. 3. This way, if there are several add-ons that want to modify the same setting, - all of them will fallback to the component from the previous one and then to + all of them will fall back to the component from the previous one and then to the Scrapy default. The order of that depends on the priority order in the ``ADDONS`` setting. @@ -166,8 +166,7 @@ Use a fallback component: .. code-block:: python - from scrapy.utils.misc import build_from_crawler - + from scrapy.utils.misc import build_from_crawler, load_object FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" @@ -175,16 +174,19 @@ Use a fallback component: class MyHandler: lazy = False - def __init__(self, settings, crawler): - dhcls = load_object(settings.get(FALLBACK_SETTING)) + def __init__(self, crawler): + dhcls = load_object(crawler.settings.get(FALLBACK_SETTING)) self._fallback_handler = build_from_crawler(dhcls, crawler) - def download_request(self, request, spider): + async def download_request(self, request): if request.meta.get("my_params"): # handle the request ... else: - return self._fallback_handler.download_request(request, spider) + return await self._fallback_handler.download_request(request) + + async def close(self): + pass class MyAddon: diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index e8c510ea5..c60c43f3c 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -63,7 +63,7 @@ this: :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`). 8. The :ref:`Engine ` sends processed items to - :ref:`Item Pipelines `, then send processed Requests to + :ref:`Item Pipelines `, then sends processed Requests to the :ref:`Scheduler ` and asks for possible next Requests to crawl. diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index ca5bc9a5c..3d4ebe088 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -4,21 +4,27 @@ asyncio ======= -.. versionadded:: 2.0 +Scrapy supports :mod:`asyncio` natively. New projects created with +:command:`startproject` have asyncio enabled by default, and you can use +:mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine +`. -Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the -asyncio reactor `, you may use :mod:`asyncio` and -:mod:`asyncio`-powered libraries in any :doc:`coroutine `. +The rest of this page covers advanced topics. If you are starting a new project, +no additional setup is needed. .. _install-asyncio: -Installing the asyncio reactor -============================== +Configuring the asyncio reactor +=============================== -To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs -to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, -which is the default value. +New projects generated with :command:`startproject` have the asyncio +reactor configured by default. No manual setup is needed. + +The :setting:`TWISTED_REACTOR` setting controls which Twisted reactor Scrapy +uses. Its default value is +``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, which enables +:mod:`asyncio` support. If you are using :class:`~scrapy.crawler.AsyncCrawlerRunner` or :class:`~scrapy.crawler.CrawlerRunner`, you also need to @@ -100,6 +106,10 @@ Scrapy API requires passing a Deferred to it) using the following helpers: .. autofunction:: scrapy.utils.defer.deferred_from_coro .. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f +The following function helps with a reverse wrapping: + +.. autofunction:: scrapy.utils.defer.ensure_awaitable + .. _enforce-asyncio-requirement: @@ -130,6 +140,173 @@ example: .. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed +.. _asyncio-without-reactor: + +Using Scrapy without a Twisted reactor +====================================== + +.. versionadded:: 2.15.0 + +.. warning:: + This is currently experimental and may not be suitable for production use. + +It's possible to use Scrapy without installing a Twisted reactor at all, by +setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this +mode Scrapy will use the asyncio event loop directly, and most of the Scrapy +functionality will work in the same way. + +Doing this provides several benefits in certain use cases: + +* A Twisted reactor, once stopped, cannot be started again. This prevents, for + example, using several instances of + :class:`~scrapy.crawler.AsyncCrawlerProcess` in the same process when they + use a reactor, but with ``TWISTED_REACTOR_ENABLED=False`` it becomes + possible. +* There may be limitations imposed by + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` and related + Twisted code, such as the requirement of using + :class:`~asyncio.SelectorEventLoop` on Windows (see :ref:`asyncio-windows`), + that do not apply if the reactor is not used. +* :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` manages the + underlying event loop, and while :class:`~scrapy.crawler.AsyncCrawlerRunner` + can use a pre-existing reactor which, in turn, can use a pre-existing event + loop, it's easier to use :class:`~scrapy.crawler.AsyncCrawlerRunner` with a + pre-existing loop directly. +* Omitting the reactor machinery may improve performance and reliability. + +Limitations +----------- + +As some Scrapy features and components require a reactor, they don't work and +are disabled without it. Replacements that don't require a reactor may be added +in future Scrapy versions. The following features are not available: + +* The default HTTP(S) download handler, + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` (this + is likely the biggest difference; Scrapy provides an HTTP(S) download handler + that doesn't require a reactor and will be used instead of it: + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`) +* :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` +* :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` +* :ref:`topics-telnetconsole` +* :class:`~scrapy.crawler.CrawlerRunner` and + :class:`~scrapy.crawler.CrawlerProcess` + (:class:`~scrapy.crawler.AsyncCrawlerProcess` and + :class:`~scrapy.crawler.AsyncCrawlerRunner` are available) +* Twisted-specific DNS resolvers (the :setting:`TWISTED_DNS_RESOLVER` setting) +* User and 3rd-party code that requires a reactor (see :ref:`below + ` for examples) + +Note that importing Twisted modules and, among other things, creating and using +:class:`~twisted.internet.defer.Deferred` objects doesn't require a reactor, so +code that uses :class:`~twisted.internet.defer.Deferred`, +:class:`~twisted.python.failure.Failure` and some other Twisted APIs will not +necessarily stop working. + +Other differences +----------------- + +When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``, Scrapy will change +the defaults of some other settings: + +* :setting:`TELNETCONSOLE_ENABLED` is set to ``False``. +* The ``"http"`` and ``"https"`` keys in :setting:`DOWNLOAD_HANDLERS_BASE` are + set to ``"scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler"``. +* The ``"ftp"`` key in :setting:`DOWNLOAD_HANDLERS_BASE` is set to ``None``. + +Thus, :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` is +used by default for making HTTP(S) requests. Please refer to its documentation +for its differences and limitations compared to +:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + +Additionally, :class:`~scrapy.crawler.AsyncCrawlerProcess` will install a +:term:`meta path finder` that prevents :mod:`twisted.internet.reactor` from +being imported. + +.. _asyncio-without-reactor-migrate: + +Adding support to existing code +------------------------------- + +Code that doesn't directly use Twisted APIs or APIs that depend on Twisted ones +doesn't need special support for running without a reactor. + +Here are some examples of APIs and patterns that need a replacement: + +* Using :meth:`reactor.callLater() + ` for sleeping or delayed calls. + You can use :meth:`asyncio.loop.call_later` instead. +* Using :func:`twisted.internet.threads.deferToThread`, + :meth:`reactor.callFromThread() + ` and related APIs to + execute code in other threads. You can use :func:`asyncio.to_thread`, + :meth:`asyncio.loop.call_soon_threadsafe` and related APIs instead. +* Using :class:`twisted.internet.task.LoopingCall` for scheduling repeated + tasks. As there is no direct replacement in the standard library, you may + need to write your own one using :func:`asyncio.sleep` in a task. +* Using Twisted network client and server APIs (:meth:`reactor.connectTCP() + `, + :meth:`reactor.listenTCP() + `, + :mod:`twisted.web.client`, :mod:`twisted.mail.smtp` etc.). You can use other + built-in or 3rd-party libraries for this. +* Using :class:`~scrapy.crawler.CrawlerProcess` or + :class:`~scrapy.crawler.CrawlerRunner`. You should use + :class:`~scrapy.crawler.AsyncCrawlerProcess` or + :class:`~scrapy.crawler.AsyncCrawlerRunner` respectively instead. +* Checking whether ``asyncio`` support is available with + :func:`scrapy.utils.reactor.is_asyncio_reactor_installed`. You should use + :func:`scrapy.utils.asyncio.is_asyncio_available` instead. + +Scrapy provides unified helpers for some of these examples: + +.. autofunction:: scrapy.utils.asyncio.call_later +.. autofunction:: scrapy.utils.asyncio.create_looping_call +.. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall +.. autofunction:: scrapy.utils.asyncio.run_in_thread + +If your code needs to know whether the reactor is available, you can either +check for the value of the :setting:`TWISTED_REACTOR_ENABLED` setting (you need +access to the :class:`~scrapy.crawler.Crawler` instance to do this) or use the +following function: + +.. autofunction:: scrapy.utils.reactorless.is_reactorless + +In general, code that doesn't use the reactor (directly or indirectly) can be +used unmodified both with the asyncio reactor and without a reactor. This +includes code that converts Deferreds to futures and vice versa as described in +:ref:`asyncio-await-dfd`. + +Troubleshooting +--------------- + +**ImportError: Import of twisted.internet.reactor is forbidden when running +without a Twisted reactor [...]:** Scrapy is configured to run without a +reactor, but some code imported :mod:`twisted.internet.reactor`, most likely +because that code needs a reactor to be used. You need to stop using this code +or set :setting:`TWISTED_REACTOR_ENABLED` back to ``True``. It's also possible +that the reactor isn't really needed but was installed due to the problem +described in :ref:`asyncio-preinstalled-reactor`, in which case it should be +enough to fix the problematic imports. + +**RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is +installed:** Scrapy is configured to run without a reactor, but a reactor is +already installed before the Scrapy code is executed. If you are trying to set +:setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings +`, it's currently unsupported. + +**RuntimeError: We expected a Twisted reactor to be installed but it isn't:** +Scrapy is configured to run with a reactor and not to install one, but a +reactor wasn't installed before the Scrapy code is executed. If you are trying +to set :setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings +`, it's currently unsupported. + +**RuntimeError: doesn't support TWISTED_REACTOR_ENABLED=False:** The +listed class cannot be used with :setting:`TWISTED_REACTOR_ENABLED` set to +``False``. There may be a replacement in the :ref:`documentation above +` or the documentation of the affected class. + + .. _asyncio-windows: Windows-specific notes @@ -141,8 +318,7 @@ implementations, :class:`~asyncio.ProactorEventLoop` (default) and :class:`~asyncio.SelectorEventLoop` works with Twisted. Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop` -automatically when you change the :setting:`TWISTED_REACTOR` setting or call -:func:`~scrapy.utils.reactor.install_reactor`. +automatically when installing the asyncio reactor. .. note:: Other libraries you use may require :class:`~asyncio.ProactorEventLoop`, e.g. because it supports @@ -150,6 +326,9 @@ automatically when you change the :setting:`TWISTED_REACTOR` setting or call them together with Scrapy on Windows (but you should be able to use them on WSL or native Linux). +.. note:: This problem doesn't apply when not using the reactor, see + :ref:`asyncio-without-reactor`. + .. _playwright: https://github.com/microsoft/playwright-python diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index d0321c906..4f28019da 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -75,7 +75,7 @@ AutoThrottle algorithm adjusts download delays based on the following rules: .. _download-latency: In Scrapy, the download latency is measured as the time elapsed between -establishing the TCP connection and receiving the HTTP headers. +sending the request and receiving the HTTP headers. Note that these latencies are very hard to measure accurately in a cooperative multitasking environment because Scrapy may be busy processing a spider @@ -88,6 +88,8 @@ server) is, and this extension builds on that premise. Prevent specific requests from triggering slot delay adjustments ================================================================ +.. versionadded:: 2.12.0 + AutoThrottle adjusts the delay of download slots based on the latencies of responses that belong to that download slot. The only exceptions are non-200 responses, which are only taken into account to increase that delay, but diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index b4d058754..cace1f883 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -41,19 +41,6 @@ efficient broad crawl. .. _broad-crawls-scheduler-priority-queue: -Use the right :setting:`SCHEDULER_PRIORITY_QUEUE` -================================================= - -Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. -It works best during single-domain crawl. It does not work well with crawling -many different domains in parallel - -To apply the recommended priority queue use: - -.. code-block:: python - - SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue" - .. _broad-crawls-concurrency: Increase concurrency diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 82712eba5..603789e2f 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -163,8 +163,8 @@ information on which commands must be run from inside projects, and which not. Also keep in mind that some commands may have slightly different behaviours when running them from inside projects. For example, the fetch command will use -spider-overridden behaviours (such as the ``user_agent`` attribute to override -the user-agent) if the url being fetched is associated with some specific +spider-overridden behaviours (such as the ``custom_settings`` attribute to +override settings) if the url being fetched is associated with some specific spider. This is intentional, as the ``fetch`` command is meant to be used to check how spiders are downloading pages. @@ -199,6 +199,7 @@ Global commands: * :command:`fetch` * :command:`view` * :command:`version` +* :command:`bench` Project-only commands: @@ -207,7 +208,6 @@ Project-only commands: * :command:`list` * :command:`edit` * :command:`parse` -* :command:`bench` .. command:: startproject @@ -233,9 +233,6 @@ genspider * Syntax: ``scrapy genspider [-t template] `` * Requires project: *no* -.. versionadded:: 2.6.0 - The ability to pass a URL instead of a domain. - Creates a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. Usage example:: @@ -312,11 +309,25 @@ Usage examples:: * parse_item $ scrapy check - [FAILED] first_spider:parse_item - >>> 'RetailPricex' field is missing + F.F. + ====================================================================== + FAIL: [first_spider] parse (@returns post-hook) + ---------------------------------------------------------------------- + Traceback (most recent call last): + ... + scrapy.exceptions.ContractFail: Returned 92 requests, expected 0..4 - [FAILED] first_spider:parse - >>> Returned 92 requests, expected 0..4 + ====================================================================== + FAIL: [first_spider] parse_item (@scrapes post-hook) + ---------------------------------------------------------------------- + Traceback (most recent call last): + ... + scrapy.exceptions.ContractFail: Missing fields: RetailPricex + + ---------------------------------------------------------------------- + Ran 4 contracts in 0.174s + + FAILED (failures=2) .. skip: end @@ -380,7 +391,7 @@ Supported options: * ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider -* ``--headers``: print the response's HTTP headers instead of the response's body +* ``--headers``: print the request's and response's HTTP headers instead of the response's body * ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) @@ -390,15 +401,19 @@ Usage examples:: [ ... html content here ... ] $ scrapy fetch --nolog --headers http://www.example.com/ - {'Accept-Ranges': ['bytes'], - 'Age': ['1263 '], - 'Connection': ['close '], - 'Content-Length': ['596'], - 'Content-Type': ['text/html; charset=UTF-8'], - 'Date': ['Wed, 18 Aug 2010 23:59:46 GMT'], - 'Etag': ['"573c1-254-48c9c87349680"'], - 'Last-Modified': ['Fri, 30 Jul 2010 15:30:18 GMT'], - 'Server': ['Apache/2.2.3 (CentOS)']} + > Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + > Accept-Language: en + > User-Agent: Scrapy/2.16.0 (+https://scrapy.org) + > Accept-Encoding: gzip, deflate, br + > + < Date: Wed, 08 Jul 2026 06:15:01 GMT + < Content-Type: text/html + < Server: cloudflare + < Last-Modified: Wed, 01 Jul 2026 17:50:18 GMT + < Allow: GET, HEAD + < Cf-Cache-Status: HIT + < Age: 8184 + < Cf-Ray: a17cf3b80eddf141-DME .. command:: view @@ -479,7 +494,7 @@ Supported options: * ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider -* ``--a NAME=VALUE``: set spider argument (may be repeated) +* ``-a NAME=VALUE``: set spider argument (may be repeated) * ``--callback`` or ``-c``: spider method to use as callback for parsing the response @@ -509,8 +524,6 @@ Supported options: * ``--output`` or ``-o``: dump scraped items to a file - .. versionadded:: 2.3 - .. skip: start Usage example:: @@ -610,7 +623,10 @@ shouldn't matter to the user running the command, but when the user :ref:`needs a non-default Twisted reactor `, it may be important. Scrapy decides which of these two classes to use based on the value of the -:setting:`TWISTED_REACTOR` setting. If the setting value is the default one +:setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED` settings. +With :setting:`TWISTED_REACTOR_ENABLED` set to ``False`` it will use +:class:`~scrapy.crawler.AsyncCrawlerProcess`. Otherwise, if the +:setting:`TWISTED_REACTOR` value is the default one (``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``), :class:`~scrapy.crawler.AsyncCrawlerProcess` will be used, otherwise :class:`~scrapy.crawler.CrawlerProcess` will be used. The :ref:`spider settings diff --git a/docs/topics/components.rst b/docs/topics/components.rst index 56f8c6498..c0df86922 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -11,12 +11,10 @@ That includes the classes that you may assign to the following settings: - :setting:`ADDONS` -- :setting:`DNS_RESOLVER` +- :setting:`TWISTED_DNS_RESOLVER` - :setting:`DOWNLOAD_HANDLERS` -- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` - - :setting:`DOWNLOADER_MIDDLEWARES` - :setting:`DUPEFILTER_CLASS` diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index fdd3dcd87..9dcd9d69c 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -4,8 +4,6 @@ Coroutines ========== -.. versionadded:: 2.0 - Scrapy :ref:`supports ` the :ref:`coroutine syntax ` (i.e. ``async def``). @@ -18,20 +16,13 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): -- The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be +- The :meth:`~scrapy.Spider.start` spider method, which *must* be defined as an :term:`asynchronous generator`. .. versionadded:: 2.13 - :class:`~scrapy.Request` callbacks. - If you are using any custom or third-party :ref:`spider middleware - `, see :ref:`sync-async-spider-middleware`. - - .. versionchanged:: 2.7 - Output of async callbacks is now processed asynchronously instead of - collecting all of it first. - - The :meth:`process_item` method of :ref:`item pipelines `. @@ -45,15 +36,9 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` - method of :ref:`spider middlewares `. - - If defined as a coroutine, it must be an :term:`asynchronous generator`. - The input ``result`` parameter is an :term:`asynchronous iterable`. - - See also :ref:`sync-async-spider-middleware` and - :ref:`universal-spider-middleware`. - - .. versionadded:: 2.7 + method of :ref:`spider middlewares `, which + *must* be defined as an :term:`asynchronous generator` except in + :ref:`universal spider middlewares `. - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method of :ref:`spider middlewares `, which *must* be @@ -63,6 +48,10 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. +- Methods of :ref:`download handlers `. + + .. versionadded:: 2.14 + .. _coroutine-deferred-apis: @@ -74,50 +63,29 @@ In addition to native coroutine APIs Scrapy has some APIs that return a function that returns a :class:`~twisted.internet.defer.Deferred` object. These APIs are also asynchronous but don't yet support native ``async def`` syntax. In the future we plan to add support for the ``async def`` syntax to these APIs -or replace them with other APIs where changing the existing ones is +or replace them with other APIs where changing the existing ones isn't possible. -The following Scrapy methods return :class:`~twisted.internet.defer.Deferred` -objects (this list is not complete as it only includes methods that we think -may be useful for user code): +These APIs have a coroutine-based implementation and a Deferred-based one: - :class:`scrapy.crawler.Crawler`: - - :meth:`~scrapy.crawler.Crawler.crawl` + - :meth:`~scrapy.crawler.Crawler.crawl_async` (coroutine-based) and + :meth:`~scrapy.crawler.Crawler.crawl` (Deferred-based): the former + may be inconvenient to use in Deferred-based code so both are available, + this may change in a future Scrapy version. - - :meth:`~scrapy.crawler.Crawler.stop` - -- :class:`scrapy.crawler.CrawlerRunner` (also inherited by - :class:`scrapy.crawler.CrawlerProcess`): - - - :meth:`~scrapy.crawler.CrawlerRunner.crawl` - - - :meth:`~scrapy.crawler.CrawlerRunner.stop` - - - :meth:`~scrapy.crawler.CrawlerRunner.join` - -- :class:`scrapy.core.engine.ExecutionEngine`: - - - :meth:`~scrapy.core.engine.ExecutionEngine.download` - -- :class:`scrapy.signalmanager.SignalManager`: - - - :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_deferred` - -- :class:`~scrapy.mail.MailSender` - - - :meth:`~scrapy.mail.MailSender.send` +- :class:`scrapy.crawler.AsyncCrawlerRunner` and its subclass + :class:`scrapy.crawler.AsyncCrawlerProcess` (coroutine-based) and + :class:`scrapy.crawler.CrawlerRunner` and its subclass + :class:`scrapy.crawler.CrawlerProcess` (Deferred-based): the former + doesn't support non-default reactors and so the latter should be used + with those. The following user-supplied methods can return :class:`~twisted.internet.defer.Deferred` objects (the methods that can also return coroutines are listed in :ref:`coroutine-support`): -- Custom download handlers (see :setting:`DOWNLOAD_HANDLERS`): - - - ``download_request()`` - - - ``close()`` - - Custom downloader implementations (see :setting:`DOWNLOADER`): - ``fetch()`` @@ -156,19 +124,11 @@ wrapping a :class:`~twisted.internet.defer.Deferred` object into a :class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for more information about this. -For example: - -- The :meth:`ExecutionEngine.download() - ` method returns a - :class:`~twisted.internet.defer.Deferred` object that fires with the - downloaded response. You can use this object directly in Deferred-based - code or convert it into a :class:`~asyncio.Future` object with - :func:`~scrapy.utils.defer.maybe_deferred_to_future`. -- A custom download handler needs to define a ``download_request()`` method - that returns a :class:`~twisted.internet.defer.Deferred` object. You can - write a method that works with Deferreds and returns one directly, or you - can write a coroutine and convert it into a function that returns a - Deferred with :func:`~scrapy.utils.defer.deferred_f_from_coro_f`. +For example: a custom scheduler needs to define an ``open()`` method that can +return a :class:`~twisted.internet.defer.Deferred` object. You can write a +method that works with Deferreds and returns one directly, or you can write a +coroutine and convert it into a function that returns a Deferred with +:func:`~scrapy.utils.defer.deferred_f_from_coro_f`. General usage @@ -244,13 +204,15 @@ This means you can use many useful Python libraries providing such code: Common use cases for asynchronous code include: * requesting data from websites, databases and other services (in - :meth:`~scrapy.spiders.Spider.start`, callbacks, pipelines and + :meth:`~scrapy.Spider.start`, callbacks, pipelines and middlewares); * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the :signal:`spider_opened` handler); -* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download` - (see :ref:`the screenshot pipeline example`). +* calling asynchronous Scrapy methods like + :meth:`ExecutionEngine.download_async() + ` (see :ref:`the + screenshot pipeline example `). .. _aio-libs: https://github.com/aio-libs @@ -307,143 +269,6 @@ You can also send multiple requests in parallel: responses = await asyncio.gather(*tasks) yield { "h1": response.css("h1::text").get(), - "price": responses[0][1].css(".price::text").get(), - "price2": responses[1][1].css(".color::text").get(), + "price": responses[0].css(".price::text").get(), + "color": responses[1].css(".color::text").get(), } - - -.. _sync-async-spider-middleware: - -Mixing synchronous and asynchronous spider middlewares -====================================================== - -.. versionadded:: 2.7 - -The output of a :class:`~scrapy.Request` callback is passed as the ``result`` -parameter to the -:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method -of the first :ref:`spider middleware ` from the -:ref:`list of active spider middlewares `. -Then the output of that ``process_spider_output`` method is passed to the -``process_spider_output`` method of the next spider middleware, and so on for -every active spider middleware. - -Scrapy supports mixing :ref:`coroutine methods ` and synchronous methods -in this chain of calls. - -However, if any of the ``process_spider_output`` methods is defined as a -synchronous method, and the previous ``Request`` callback or -``process_spider_output`` method is a coroutine, there are some drawbacks to -the asynchronous-to-synchronous conversion that Scrapy does so that the -synchronous ``process_spider_output`` method gets a synchronous iterable as its -``result`` parameter: - -- The whole output of the previous ``Request`` callback or - ``process_spider_output`` method is awaited at this point. - -- If an exception raises while awaiting the output of the previous - ``Request`` callback or ``process_spider_output`` method, none of that - output will be processed. - - This contrasts with the regular behavior, where all items yielded before - an exception raises are processed. - -Asynchronous-to-synchronous conversions are supported for backward -compatibility, but they are deprecated and will stop working in a future -version of Scrapy. - -To avoid asynchronous-to-synchronous conversions, when defining ``Request`` -callbacks as coroutine methods or when using spider middlewares whose -``process_spider_output`` method is an :term:`asynchronous generator`, all -active spider middlewares must either have their ``process_spider_output`` -method defined as an asynchronous generator or :ref:`define a -process_spider_output_async method `. - -.. _sync-async-spider-middleware-users: - -For middleware users --------------------- - -If you have asynchronous callbacks or use asynchronous-only spider middlewares -you should make sure the asynchronous-to-synchronous conversions -:ref:`described above ` don't happen. To do this, -make sure all spider middlewares you use support asynchronous spider output. -Even if you don't have asynchronous callbacks and don't use asynchronous-only -spider middlewares in your project, it's still a good idea to make sure all -middlewares you use support asynchronous spider output, so that it will be easy -to start using asynchronous callbacks in the future. Because of this, Scrapy -logs a warning when it detects a synchronous-only spider middleware. - -If you want to update middlewares you wrote, see the :ref:`following section -`. If you have 3rd-party middlewares that -aren't yet updated by their authors, you can :ref:`subclass ` -them to make them :ref:`universal ` and use the -subclasses in your projects. - -.. _sync-async-spider-middleware-authors: - -For middleware authors ----------------------- - -If you have a spider middleware that defines a synchronous -``process_spider_output`` method, you should update it to support asynchronous -spider output for :ref:`better compatibility `, -even if you don't yet use it with asynchronous callbacks, especially if you -publish this middleware for other people to use. You have two options for this: - -1. Make the middleware asynchronous, by making the ``process_spider_output`` - method an :term:`asynchronous generator`. -2. Make the middleware universal, as described in the :ref:`next section - `. - -If your middleware won't be used in projects with synchronous-only middlewares, -e.g. because it's an internal middleware and you know that all other -middlewares in your projects are already updated, it's safe to choose the first -option. Otherwise, it's better to choose the second option. - -.. _universal-spider-middleware: - -Universal spider middlewares ----------------------------- - -.. versionadded:: 2.7 - -To allow writing a spider middleware that supports asynchronous execution of -its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding -:ref:`asynchronous-to-synchronous conversions `) -while maintaining support for older Scrapy versions, you may define -``process_spider_output`` as a synchronous method and define an -:term:`asynchronous generator` version of that method with an alternative name: -``process_spider_output_async``. - -For example: - -.. code-block:: python - - class UniversalSpiderMiddleware: - def process_spider_output(self, response, result): - for r in result: - # ... do something with r - yield r - - async def process_spider_output_async(self, response, result): - async for r in result: - # ... do something with r - yield r - -.. note:: This is an interim measure to allow, for a time, to write code that - works in Scrapy 2.7 and later without requiring - asynchronous-to-synchronous conversions, and works in earlier Scrapy - versions as well. - - In some future version of Scrapy, however, this feature will be - deprecated and, eventually, in a later version of Scrapy, this - feature will be removed, and all spider middlewares will be expected - to define their ``process_spider_output`` method as an asynchronous - generator. - -Since 2.13.0, Scrapy provides a base class, -:class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which implements -the ``process_spider_output()`` and ``process_spider_output_async()`` methods, -so instead of duplicating the processing code you can override the -``get_processed_request()`` and/or the ``get_processed_item()`` method. diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst new file mode 100644 index 000000000..5fbc5df06 --- /dev/null +++ b/docs/topics/download-handlers.rst @@ -0,0 +1,393 @@ +.. _topics-download-handlers: + +================= +Download handlers +================= + +Download handlers are Scrapy :ref:`components ` used to +download :ref:`requests ` and produce responses from +them. + +Using download handlers +======================= + +The :setting:`DOWNLOAD_HANDLERS_BASE` and :setting:`DOWNLOAD_HANDLERS` settings +tell Scrapy which handler is responsible for a given URL scheme. Their values +are merged into a mapping from scheme names to handler classes. When Scrapy +initializes it creates instances of all configured download handlers (except +for :ref:`lazy ones `) and stores them in a similar +mapping. When Scrapy needs to download a request it extracts the scheme from +its URL, finds the handler for this scheme, passes the request to it and gets a +response from it. If there is no handler for the scheme, the request is not +downloaded and a :exc:`~scrapy.exceptions.NotSupported` exception is raised. + +The :setting:`DOWNLOAD_HANDLERS_BASE` setting contains the default mapping of +handlers. You can use the :setting:`DOWNLOAD_HANDLERS` setting to add handlers +for additional schemes and to replace or disable default ones: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + # disable support for ftp:// requests + "ftp": None, + # replace the default one for http:// + "http": "my.download_handlers.HttpHandler", + # http:// and https:// are different schemes, + # even though they may use the same handler + "https": "my.download_handlers.HttpHandler", + # support for any custom scheme can be added + "sftp": "my.download_handlers.SftpHandler", + } + +.. seealso:: :ref:`security-unencrypted-protocols` and + :ref:`security-local-resources`, for the security implications of the + default ``http``, ``ftp``, ``file`` and ``data`` handlers. + +Replacing HTTP(S) download handlers +----------------------------------- + +While Scrapy provides a default handler for ``http`` and ``https`` schemes, +users may want to use a different handler, provided by Scrapy or by some +3rd-party package. There are several considerations to keep in mind related to +this. + +First of all, as ``http`` and ``https`` are separate schemes, they need +separate entries in the :setting:`DOWNLOAD_HANDLERS` setting, even though it's +likely that the same handler class will be used for both schemes. + +Additionally, some of the Scrapy settings, like :setting:`DOWNLOAD_MAXSIZE`, +are honored by the default HTTP(S) handler but not necessarily by alternative +ones. The same may apply to other Scrapy features, e.g. the +:signal:`bytes_received` and :signal:`headers_received` signals. + +.. _lazy-download-handlers: + +Lazy instantiation of download handlers +--------------------------------------- + +A download handler can be marked as "lazy" by setting its ``lazy`` class +attribute to ``True``. Such handlers are only instantiated when they need to +download their first request. This may be useful when the instantiation is slow +or requires dependencies that are not always available, and the handler is not +needed on every spider run. For example, :class:`the built-in S3 handler +<.S3DownloadHandler>` is lazy. + +Writing your own download handler +================================= + +A download handler is a :ref:`component ` that defines +the following API: + +.. class:: SampleDownloadHandler + + .. attribute:: lazy + :type: bool + + If ``False``, the handler will be instantiated when Scrapy is + initialized. + + If ``True``, the handler will only be instantiated when the first + request handled by it needs to be downloaded. + + .. method:: download_request(request: Request) -> Response + :async: + + Download the given request and return a response. + + .. method:: close() -> None + :async: + + Clean up any resources used by the handler. + +An optional base class for custom handlers is provided: + +.. autoclass:: scrapy.core.downloader.handlers.base.BaseDownloadHandler + :members: + :undoc-members: + :member-order: bysource + +.. _download-handlers-exceptions: + +Exceptions raised by download handlers +====================================== + +.. versionadded:: 2.15.0 + +The built-in download handlers raise Scrapy-specific exceptions instead of +implementation-specific ones, so that code that handles these exceptions can be +written in a generic way. We recommend custom download handlers to also use +these exceptions. + +.. autoexception:: scrapy.exceptions.CannotResolveHostError + +.. autoexception:: scrapy.exceptions.DownloadCancelledError + +.. autoexception:: scrapy.exceptions.DownloadConnectionRefusedError + +.. autoexception:: scrapy.exceptions.DownloadFailedError + +.. autoexception:: scrapy.exceptions.DownloadTimeoutError + +.. autoexception:: scrapy.exceptions.ResponseDataLossError + +.. autoexception:: scrapy.exceptions.UnsupportedURLSchemeError + +.. _download-handlers-ref: + +Built-in HTTP download handlers reference +========================================= + +Scrapy ships several handlers for HTTP and HTTPS requests. While all of them +support basic features, they may differ in support of specific Scrapy features +and settings and HTTP protocol features. See the documentation of specific +handlers and specific settings for more information. Additionally, as the +underlying HTTP client implementations differ between handlers, the behavior of +specific websites may be different when doing the same Scrapy requests but +using different handlers. + +Here is a comparison of some features of the built-in HTTP handlers, see the +individual handler docs for more differences: + +================== ================= ===================== ==================== +Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler +================== ================= ===================== ==================== +Requires asyncio No No Yes +Requires a reactor Yes Yes No +HTTP/1.1 No Yes Yes +HTTP/2 Yes No Yes +TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl`` +HTTP proxies No Yes Yes +SOCKS proxies No No Yes +================== ================= ===================== ==================== + +You can find additional HTTP download handlers in the +scrapy-download-handlers-incubator_ package. This package is made by the Scrapy +developers and contains experimental handlers that may be included in some +later Scrapy version but can already be used. Please refer to the documentation +of this package for more information. + +.. _scrapy-download-handlers-incubator: https://github.com/scrapy-plugins/scrapy-download-handlers-incubator + +.. _twisted-http2-handler: + +H2DownloadHandler +----------------- + +.. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler + +| Supported scheme: ``https``. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocol +for them. + +It's implemented using :mod:`twisted.web.client` and the ``h2`` library. + +For this handler to work you need to install the ``Twisted[http2]`` extra +dependency. + +If you want to use this handler you need to replace the default one for the +``https`` scheme: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler", + } + +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. warning:: + + This handler is experimental, and not yet recommended for production + environments. Future Scrapy versions may introduce related changes without + a deprecation period or warning. + +=========================== ================================================ +HTTP proxies No (not implemented) +SOCKS proxies No (not supported by the library) +HTTP/2 Yes +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ + +Other limitations: + +- No support for HTTP/1.1. + +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. + +- No support for the :signal:`bytes_received` and :signal:`headers_received` + signals. + +Known limitations of the HTTP/2 support: + +- No support for HTTP/2 Cleartext (h2c), since no major browser supports + HTTP/2 unencrypted (refer `http2 faq`_). + +- No setting to specify a maximum `frame size`_ larger than the default + value, 16384. Connections to servers that send a larger frame will fail. + +- No support for `server pushes`_, which are ignored. + +.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2 +.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption +.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2 + +HTTP11DownloadHandler +--------------------- + +.. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler + +| Supported schemes: ``http``, ``https``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``http://host/path`` and ``https://host/path`` URLs and +uses the HTTP/1.1 protocol for them. + +It's implemented using :mod:`twisted.web.client`. + +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +=========================== ================================================ +HTTP proxies Yes +SOCKS proxies No (not supported by the library) +HTTP/2 No (implemented as a separate handler) +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ + +Other limitations: + +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. + +- HTTPS proxies to HTTPS destinations are not supported. + +HttpxDownloadHandler +-------------------- + +.. versionadded:: 2.15.0 + +.. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler + +| Supported schemes: ``http``, ``https``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: yes. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``http://host/path`` and ``https://host/path`` URLs and +uses the HTTP/1.1 or HTTP/2 protocol for them. + +It's implemented using the ``httpx`` library and needs it to be installed. + +If you want to use this handler you need to replace the default ones for the +``http`` and ``https`` schemes: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + } + +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. warning:: + + This handler is experimental, and not yet recommended for production + environments. Future Scrapy versions may introduce related changes without + a deprecation period or warning or even remove it altogether. + +=========================== ======================================= +HTTP proxies Yes +SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``) +HTTP/2 Yes (requires ``httpx[http2]``) +``response.certificate`` DER bytes +Per-request ``bindaddress`` No (not supported by the library) +TLS implementation Standard library ``ssl`` +=========================== ======================================= + +Other limitations: + +- The handler creates a separate connection pool for each proxy URL (due to + limitations of ``httpx``) which may lead to higher resource usage when + using proxy rotation. + +.. setting:: HTTPX_HTTP2_ENABLED + +HTTPX_HTTP2_ENABLED +^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 2.17.0 + +Default: ``False`` + +Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra +needs to be installed if you want to enable this setting. + +Built-in non-HTTP download handlers reference +============================================= + +DataURIDownloadHandler +---------------------- + +.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler + +| Supported scheme: ``data``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports RFC 2397 ``data:content/type;base64,`` data URIs. + +FileDownloadHandler +------------------- + +.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler + +| Supported scheme: ``file``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``file:///path`` local file URIs. It doesn't +support remote files. + +FTPDownloadHandler +------------------ + +.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler + +| Supported scheme: ``ftp``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``ftp://host/path`` FTP URIs. + +It's implemented using :mod:`twisted.protocols.ftp`. + +S3DownloadHandler +----------------- + +.. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler + +| Supported scheme: ``s3``. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``s3://bucket/path`` S3 URIs. + +It's implemented using the ``botocore`` library and needs it to be installed. diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 6f36acc68..096f87edd 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -68,7 +68,8 @@ defines one or more of these methods: .. class:: DownloaderMiddleware - .. note:: Any of the downloader middleware methods may also return a deferred. + .. note:: Any of the downloader middleware methods may be defined as a + coroutine function (``async def``). .. method:: process_request(request) @@ -128,9 +129,10 @@ defines one or more of these methods: .. method:: process_exception(request, exception) - Scrapy calls :meth:`process_exception` when a download handler - or a :meth:`process_request` (from a downloader middleware) raises an - exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception) + Scrapy calls :meth:`process_exception` when a :ref:`download handler + ` or a :meth:`process_request` (from a + downloader middleware) raises an exception (including an + :exc:`~scrapy.exceptions.IgnoreRequest` exception). :meth:`process_exception` should return: either ``None``, a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object. @@ -289,13 +291,12 @@ DownloadTimeoutMiddleware .. class:: DownloadTimeoutMiddleware This middleware sets the download timeout for requests specified in the - :setting:`DOWNLOAD_TIMEOUT` setting or :attr:`download_timeout` - spider attribute. + :setting:`DOWNLOAD_TIMEOUT` setting. .. note:: - You can also set download timeout per-request using - :reqmeta:`download_timeout` Request.meta key; this is supported + You can also set download timeout per-request using the + :reqmeta:`download_timeout` :attr:`.Request.meta` key; this is supported even when DownloadTimeoutMiddleware is disabled. HttpAuthMiddleware @@ -306,26 +307,15 @@ HttpAuthMiddleware .. class:: HttpAuthMiddleware - This middleware authenticates all requests generated from certain spiders - using `Basic access authentication`_ (aka. HTTP auth). + This middleware authenticates requests using `Basic access authentication`_ + (aka. HTTP auth). - To enable HTTP authentication for a spider, set the ``http_user`` and - ``http_pass`` spider attributes to the authentication data and the - ``http_auth_domain`` spider attribute to the domain which requires this - authentication (its subdomains will be also handled in the same way). - You can set ``http_auth_domain`` to ``None`` to enable the - authentication for all requests but you risk leaking your authentication - credentials to unrelated domains. + Use the :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS`, and + :setting:`HTTPAUTH_DOMAIN` settings to configure it. You can also override + the credentials per request via :attr:`~scrapy.Request.meta` keys + :reqmeta:`http_user`, :reqmeta:`http_pass`, and :reqmeta:`http_auth_domain`. - .. warning:: - In previous Scrapy versions HttpAuthMiddleware sent the authentication - data with all requests, which is a security problem if the spider - makes requests to several different domains. Currently if the - ``http_auth_domain`` attribute is not set, the middleware will use the - domain of the first request, which will work for some spiders but not - for others. In the future the middleware will produce an error instead. - - Example: + Example using settings (e.g. in :attr:`~scrapy.Spider.custom_settings`): .. code-block:: python @@ -333,13 +323,70 @@ HttpAuthMiddleware class SomeIntranetSiteSpider(CrawlSpider): - http_user = "someuser" - http_pass = "somepass" - http_auth_domain = "intranet.example.com" name = "intranet.example.com" + custom_settings = { + "HTTPAUTH_USER": "someuser", + "HTTPAUTH_PASS": "somepass", + "HTTPAUTH_DOMAIN": "intranet.example.com", + } # .. rest of the spider code omitted ... + Example using per-request meta: + + .. code-block:: python + + async def start(self): + yield Request( + "https://intranet.example.com/protected/", + meta={ + "http_user": "someuser", + "http_pass": "somepass", + "http_auth_domain": "intranet.example.com", + }, + ) + +.. setting:: HTTPAUTH_USER + +HTTPAUTH_USER +~~~~~~~~~~~~~ + +.. versionadded:: 2.17.0 + +Default: ``""`` + +The username to use for HTTP basic authentication, applied to all requests +whose URL matches :setting:`HTTPAUTH_DOMAIN`. + +.. setting:: HTTPAUTH_PASS + +HTTPAUTH_PASS +~~~~~~~~~~~~~ + +.. versionadded:: 2.17.0 + +Default: ``""`` + +The password to use for HTTP basic authentication. + +.. setting:: HTTPAUTH_DOMAIN + +HTTPAUTH_DOMAIN +~~~~~~~~~~~~~~~ + +.. versionadded:: 2.17.0 + +Default: ``None`` + +The domain (and its subdomains) to which HTTP basic authentication credentials +are sent. Set to ``None`` to send credentials with all requests, but be aware +that this risks leaking credentials to unrelated domains. + +This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER` +or :setting:`HTTPAUTH_PASS` is set. + +.. seealso:: :ref:`security-credential-leakage` + .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication @@ -458,7 +505,7 @@ Filesystem storage backend (default) * ``response_body`` - the plain response body - * ``response_headers`` - the request headers (in raw HTTP format) + * ``response_headers`` - the response headers (in raw HTTP format) * ``meta`` - some metadata of this cache resource in Python ``repr()`` format (grep-friendly format) @@ -500,7 +547,7 @@ defines the methods described below. .. method:: open_spider(spider) This method gets called after a spider has been opened for crawling. It handles - the :signal:`open_spider ` signal. + the :signal:`spider_opened` signal. :param spider: the spider which has been opened :type spider: :class:`~scrapy.Spider` object @@ -508,7 +555,7 @@ defines the methods described below. .. method:: close_spider(spider) This method gets called after a spider has been closed. It handles - the :signal:`close_spider ` signal. + the :signal:`spider_closed` signal. :param spider: the spider which has been closed :type spider: :class:`~scrapy.Spider` object @@ -544,8 +591,8 @@ In order to use your storage backend, set: HTTPCache middleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The :class:`HttpCacheMiddleware` can be configured through the following -settings: +:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` can be +configured through the following settings: .. setting:: HTTPCACHE_ENABLED @@ -725,7 +772,7 @@ HttpProxyMiddleware .. class:: HttpProxyMiddleware This middleware sets the HTTP proxy to use for requests, by setting the - ``proxy`` meta value for :class:`~scrapy.Request` objects. + :reqmeta:`proxy` meta value for :class:`~scrapy.Request` objects. Like the Python standard library module :mod:`urllib.request`, it obeys the following environment variables: @@ -734,16 +781,39 @@ HttpProxyMiddleware * ``https_proxy`` * ``no_proxy`` - You can also set the meta key ``proxy`` per-request, to a value like + You can also set the meta key :reqmeta:`proxy` per-request, to a value like ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``. Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. +.. note:: + + Handling of this meta key needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. It's currently unsupported by + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`. + +.. note:: + + Usually a proxy URL uses the ``http://`` scheme. More rarely, it uses the + ``https://`` one. While both kinds of proxy URLs can be used with both HTTP + and HTTPS destination URLs, the specifics of the network exchange are + different for all 4 cases and it's possible that HTTPS proxies are fully or + partially unsupported by a given download handler. Currently, + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + supports HTTPS proxies only for HTTP destinations. + +.. note:: + + If the download handler supports it, you can use a SOCKS proxy URL (e.g. + ``socks5://username:password@some_proxy_server:port``). + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` + supports SOCKS proxies while other built-in handlers don't. + HttpProxyMiddleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. setting:: HTTPPROXY_ENABLED -.. setting:: HTTPPROXY_AUTH_ENCODING HTTPPROXY_ENABLED ^^^^^^^^^^^^^^^^^ @@ -752,6 +822,8 @@ Default: ``True`` Whether or not to enable the :class:`HttpProxyMiddleware`. +.. setting:: HTTPPROXY_AUTH_ENCODING + HTTPPROXY_AUTH_ENCODING ^^^^^^^^^^^^^^^^^^^^^^^ @@ -796,9 +868,9 @@ OffsiteMiddleware .. reqmeta:: allow_offsite If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to - ``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then - the OffsiteMiddleware will allow the request even if its domain is not listed - in allowed domains. + ``True`` or :attr:`Request.meta ` has ``allow_offsite`` + set to ``True``, then the OffsiteMiddleware will allow the request even if + its domain is not listed in allowed domains. RedirectMiddleware ------------------ @@ -913,14 +985,10 @@ Whether the Meta Refresh middleware will be enabled. METAREFRESH_IGNORE_TAGS ^^^^^^^^^^^^^^^^^^^^^^^ -Default: ``[]`` +Default: ``["noscript"]`` Meta tags within these tags are ignored. -.. versionchanged:: 2.0 - The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from - ``["script", "noscript"]`` to ``[]``. - .. versionchanged:: 2.11.2 The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from ``[]`` to ``["noscript"]``. @@ -947,17 +1015,6 @@ RetryMiddleware A middleware to retry failed requests that are potentially caused by temporary problems such as a connection timeout or HTTP 500 error. -Failed pages are collected on the scraping process and rescheduled at the -end, once the spider has finished crawling all regular (non failed) pages. - -The :class:`RetryMiddleware` can be configured through the following -settings (see the settings documentation for more info): - -* :setting:`RETRY_ENABLED` -* :setting:`RETRY_TIMES` -* :setting:`RETRY_HTTP_CODES` -* :setting:`RETRY_EXCEPTIONS` - .. reqmeta:: dont_retry If :attr:`Request.meta ` has ``dont_retry`` key @@ -1016,16 +1073,15 @@ RETRY_EXCEPTIONS Default:: [ - 'twisted.internet.defer.TimeoutError', - 'twisted.internet.error.TimeoutError', - 'twisted.internet.error.DNSLookupError', - 'twisted.internet.error.ConnectionRefusedError', + 'scrapy.exceptions.CannotResolveHostError', + 'scrapy.exceptions.DownloadConnectionRefusedError', + 'scrapy.exceptions.DownloadFailedError', + 'scrapy.exceptions.DownloadTimeoutError', + 'scrapy.exceptions.ResponseDataLossError', 'twisted.internet.error.ConnectionDone', 'twisted.internet.error.ConnectError', 'twisted.internet.error.ConnectionLost', - 'twisted.internet.error.TCPTimedOutError', - 'twisted.web.client.ResponseFailed', - IOError, + OSError, 'scrapy.core.downloader.handlers.http11.TunnelError', ] @@ -1039,6 +1095,23 @@ has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught exception propagation, see :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`. +.. setting:: RETRY_GIVE_UP_LOG_LEVEL + +RETRY_GIVE_UP_LOG_LEVEL +^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 2.17.0 + +Default: ``"ERROR"`` + +:ref:`Logging level ` used for the message logged when a request +exceeds its retries. + +Can be a level name (e.g. ``"WARNING"``) or a number (e.g. ``logging.WARNING`` +or ``30``). + +See also: :reqmeta:`give_up_log_level`, :func:`get_retry_request`. + .. setting:: RETRY_PRIORITY_ADJUST RETRY_PRIORITY_ADJUST @@ -1100,7 +1173,7 @@ Parsers vary in several aspects: * Support for wildcard matching -* Usage of `length based rule `_: +* Usage of `length based rule `_: in particular for ``Allow`` and ``Disallow`` directives, where the most specific rule based on the length of the path trumps the less specific (shorter) rule @@ -1118,7 +1191,7 @@ Based on `Protego `_: * implemented in Python * is compliant with `Google's Robots.txt Specification - `_ + `_ * supports wildcard matching @@ -1138,9 +1211,9 @@ Based on :class:`~urllib.robotparser.RobotFileParser`: * is compliant with `Martijn Koster's 1996 draft specification `_ -* lacks support for wildcard matching +* lacks support for wildcard matching (before Python 3.14.5) -* doesn't use the length based rule +* doesn't use the length based rule (before Python 3.14.5) It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0. @@ -1211,9 +1284,8 @@ UserAgentMiddleware .. class:: UserAgentMiddleware - Middleware that allows spiders to override the default user agent. + Middleware that sets the ``User-Agent`` header. - In order for a spider to override the default user agent, its ``user_agent`` - attribute must be set. + The header value is taken from the :setting:`USER_AGENT` setting. .. _DBM: https://en.wikipedia.org/wiki/Dbm diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 18b3ce244..30b6536c7 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -83,7 +83,7 @@ request with Scrapy. It might be enough to yield a :class:`~scrapy.Request` with the same HTTP method and URL. However, you may also need to reproduce the body, headers and -form parameters (see :class:`~scrapy.FormRequest`) of that request. +form parameters (see :ref:`form`) of that request. As all major browsers allow to export the requests in curl_ format, Scrapy incorporates the method :meth:`~scrapy.Request.from_curl` to generate an equivalent @@ -133,7 +133,7 @@ data from it depends on the type of response: .. code-block:: python - selector = Selector(data["html"]) + selector = Selector(text=data["html"]) - If the response is JavaScript, or HTML with a ``