diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index a9fc3dd68..1f062eef2 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -4,4 +4,4 @@ e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d # reapplying black to the code with default line length 303f0a70fcf8067adf0a909c2096a5009162383a # reapplying black again and removing line length on pre-commit black config -c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962 \ No newline at end of file +c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..98a74f8ce --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,31 @@ + diff --git a/.github/workflows/auto-close-llm-pr.yml b/.github/workflows/auto-close-llm-pr.yml new file mode 100644 index 000000000..160b39488 --- /dev/null +++ b/.github/workflows/auto-close-llm-pr.yml @@ -0,0 +1,48 @@ +name: Auto-close LLM PRs +on: + pull_request_target: + types: [opened] +permissions: + contents: read + pull-requests: write +jobs: + close-llm-pr: + name: Close PR if marked as LLM-written + runs-on: ubuntu-latest + steps: + - name: Check PR body and close if LLM-written + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const marker = "This PR was written entirely using an LLM"; + const { owner, repo } = context.repo; + const prNumber = context.payload.pull_request && context.payload.pull_request.number; + if (!prNumber) { + console.log('No pull request number found in context; exiting.'); + return; + } + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber }); + const body = pr.body || ""; + if (body.includes(marker)) { + if (pr.state === 'closed') { + console.log(`PR #${prNumber} already closed.`); + return; + } + await github.rest.issues.addLabels({ + owner, + repo, + issue_number: prNumber, + labels: ['spam'] + }); + await github.rest.issues.createComment({ + owner, + repo, + issue_number: prNumber, + body: "Closing this PR because it contains the disclosure: \"This PR was written entirely using an LLM\"." + }); + await github.rest.pulls.update({ owner, repo, pull_number: prNumber, state: 'closed' }); + console.log(`Closed PR #${prNumber} because marker was found.`); + } else { + console.log(`Marker not found in PR #${prNumber}; nothing to do.`); + } diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index a064bf5b2..ed2388a59 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,5 +1,10 @@ name: Checks -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} @@ -12,27 +17,31 @@ jobs: fail-fast: false matrix: include: - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: pylint - - python-version: "3.9" + - python-version: "3.10" env: - TOXENV: typing - - python-version: "3.9" + TOXENV: mypy + - python-version: "3.10" env: - TOXENV: typing-tests - - python-version: "3.13" # Keep in sync with .readthedocs.yml + TOXENV: mypy-tests + # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version. + - python-version: "3.14" env: TOXENV: docs - python-version: "3.13" + env: + TOXENV: docs-tests + - python-version: "3.14" env: TOXENV: twinecheck steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} @@ -45,5 +54,5 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8e01ffd88..7779bbb6b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -18,11 +18,11 @@ jobs: permissions: id-token: write steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 with: - python-version: "3.13" - - run: | + python-version: "3.14" + - run: | python -m pip install --upgrade build python -m build - name: Publish to PyPI diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml index c28a99982..0409b3ef2 100644 --- a/.github/workflows/tests-macos.yml +++ b/.github/workflows/tests-macos.yml @@ -1,5 +1,10 @@ name: macOS -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} @@ -8,23 +13,38 @@ concurrency: jobs: tests: runs-on: macos-latest + env: + PYTEST_ADDOPTS: -n auto strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + env: + - TOXENV: py + include: + - python-version: '3.14' + env: + TOXENV: no-reactor steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Run tests + env: ${{ matrix.env }} run: | pip install -U tox - tox -e py + tox - name: Upload coverage report uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/codecov-action@v5 + with: + report_type: test_results diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 89d1e70ac..15f25d9b8 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -1,5 +1,10 @@ name: Ubuntu -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} @@ -8,13 +13,12 @@ concurrency: jobs: tests: runs-on: ubuntu-latest + env: + PYTEST_ADDOPTS: -n auto strategy: fail-fast: false matrix: include: - - python-version: "3.9" - env: - TOXENV: py - python-version: "3.10" env: TOXENV: py @@ -27,53 +31,71 @@ jobs: - python-version: "3.13" env: TOXENV: py - - python-version: "3.13" + - python-version: "3.14" env: - TOXENV: asyncio - - python-version: pypy3.9 + TOXENV: py + - python-version: "3.14" env: - TOXENV: pypy3 - - python-version: pypy3.10 + TOXENV: default-reactor + - python-version: "3.14" + env: + TOXENV: no-reactor + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 env: TOXENV: pypy3 - # pinned deps - - python-version: 3.9.19 + # min deps + - python-version: "3.10.19" env: - TOXENV: pinned - - python-version: 3.9.19 + TOXENV: min + - python-version: "3.10.19" env: - TOXENV: asyncio-pinned - - python-version: pypy3.9 + TOXENV: min-default-reactor + - python-version: "3.10.19" env: - TOXENV: pypy3-pinned - - python-version: 3.9.19 + TOXENV: min-no-reactor + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 env: - TOXENV: extra-deps-pinned - - python-version: 3.9.19 + TOXENV: min-pypy3 + - python-version: "3.10.19" env: - TOXENV: botocore-pinned + TOXENV: min-extra-deps + - python-version: "3.10.19" + env: + TOXENV: min-botocore - - python-version: "3.13" + - python-version: "3.14" env: TOXENV: extra-deps - - python-version: "3.13" + - python-version: "3.14" + env: + TOXENV: no-reactor-extra-deps + # pinned due to https://github.com/pypy/pypy/issues/5388 + - python-version: pypy3.11-7.3.20 + env: + TOXENV: pypy3-extra-deps + - python-version: "3.14" env: TOXENV: botocore steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - name: Install system libraries - if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned') + if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'min') run: | sudo apt-get update - sudo apt-get install libxml2-dev libxslt-dev libjpeg-dev + sudo apt-get install libxml2-dev libxslt-dev + + - name: Install mitmproxy + run: pipx install mitmproxy - name: Run tests env: ${{ matrix.env }} @@ -83,3 +105,9 @@ jobs: - name: Upload coverage report uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/codecov-action@v5 + with: + report_type: test_results diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 45e4ca157..f413782bc 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -1,5 +1,10 @@ name: Windows -on: [push, pull_request] +on: + push: + branches: + - master + - '[0-9]+.[0-9]+' + pull_request: concurrency: group: ${{github.workflow}}-${{ github.ref }} @@ -8,13 +13,12 @@ concurrency: jobs: tests: runs-on: windows-latest + env: + PYTEST_ADDOPTS: -n auto strategy: fail-fast: false matrix: include: - - python-version: "3.9" - env: - TOXENV: windows-pinned - python-version: "3.10" env: TOXENV: py @@ -27,15 +31,33 @@ jobs: - python-version: "3.13" env: TOXENV: py - - python-version: "3.13" + - python-version: "3.14" env: - TOXENV: asyncio + TOXENV: py + - python-version: "3.14" + env: + TOXENV: default-reactor + - python-version: "3.14" + env: + TOXENV: no-reactor + + # min deps + - python-version: "3.10.11" + env: + TOXENV: min + - python-version: "3.10.11" + env: + TOXENV: min-extra-deps + + - python-version: "3.14" + env: + TOXENV: extra-deps steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} @@ -47,3 +69,9 @@ jobs: - name: Upload coverage report uses: codecov/codecov-action@v5 + + - name: Upload test results + if: ${{ !cancelled() }} + uses: codecov/codecov-action@v5 + with: + report_type: test_results diff --git a/.gitignore b/.gitignore index 6c5c50e08..5e52ecf1e 100644 --- a/.gitignore +++ b/.gitignore @@ -3,18 +3,21 @@ *.pyc _trial_temp* dropin.cache -docs/build +docs/_build *egg-info -.tox -venv -build -dist -.idea +.tox/ +venv/ +.venv/ +build/ +dist/ +.idea/ +.vscode/ htmlcov/ -.coverage .pytest_cache/ +.coverage .coverage.* coverage.* +*.junit.xml test-output.* .cache/ .mypy_cache/ @@ -25,4 +28,4 @@ test-output.* Thumbs.db # OSX miscellaneous -.DS_Store \ No newline at end of file +.DS_Store diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c76c613d9..311df7052 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,32 @@ +exclude: | + (?x)( + ^docs/_static| + ^docs/_tests| + ^tests/sample_data + ) repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.8.4 + rev: v0.15.20 hooks: - - id: ruff + - id: ruff-check args: [ --fix ] -- repo: https://github.com/psf/black.git - rev: 24.10.0 - hooks: - - id: black + - id: ruff-format - repo: https://github.com/adamchainz/blacken-docs - rev: 1.19.1 + rev: 1.20.0 hooks: - id: blacken-docs additional_dependencies: - - black==24.10.0 + - black==26.5.1 +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace +- repo: https://github.com/sphinx-contrib/sphinx-lint + rev: v1.0.2 + hooks: + - id: sphinx-lint +- repo: https://github.com/scrapy/sphinx-scrapy + rev: 0.8.8 + hooks: + - id: sphinx-scrapy diff --git a/.readthedocs.yml b/.readthedocs.yml index 5ec6eafbb..a2773dcf2 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,17 +1,10 @@ version: 2 -formats: all -sphinx: - configuration: docs/conf.py - fail_on_warning: true - build: - os: ubuntu-20.04 + os: ubuntu-24.04 tools: - # For available versions, see: - # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python - python: "3.13" # Keep in sync with .github/workflows/checks.yml - -python: - install: - - requirements: docs/requirements.txt - - path: . + python: "3.14" + commands: + - pip install tox + - tox -e docs + - mkdir -p $READTHEDOCS_OUTPUT/html + - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/ diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..24a426d36 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,6 @@ +cff-version: 1.2.0 +message: If you use Scrapy in published research, please cite it as below. +title: Scrapy +authors: + - name: Scrapy contributors +url: https://scrapy.org diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 7700ae7bd..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,22 +0,0 @@ -include CODE_OF_CONDUCT.md -include CONTRIBUTING.md -include INSTALL.md -include NEWS -include SECURITY.md - -include scrapy/VERSION -include scrapy/mime.types -include scrapy/py.typed - -include codecov.yml -include conftest.py -include tox.ini - -recursive-include scrapy/templates * -recursive-include docs * -prune docs/build - -recursive-include extras * -recursive-include tests * - -global-exclude __pycache__ *.py[cod] diff --git a/README.rst b/README.rst index 3f468953e..6235cb20c 100644 --- a/README.rst +++ b/README.rst @@ -1,114 +1,62 @@ -.. image:: https://scrapy.org/img/scrapylogo.png - :target: https://scrapy.org/ - -====== -Scrapy -====== +|logo| -.. image:: https://img.shields.io/pypi/v/Scrapy.svg +.. |logo| image:: https://raw.githubusercontent.com/scrapy/scrapy/master/docs/_static/logo.svg + :target: https://scrapy.org + :alt: Scrapy + :width: 480px + +|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki| + +.. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg :target: https://pypi.org/pypi/Scrapy :alt: PyPI Version -.. image:: https://img.shields.io/pypi/pyversions/Scrapy.svg +.. |python_version| image:: https://img.shields.io/pypi/pyversions/Scrapy.svg :target: https://pypi.org/pypi/Scrapy :alt: Supported Python Versions -.. image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg +.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu :alt: Ubuntu -.. .. image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg - .. :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS - .. :alt: macOS +.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg + :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS + :alt: macOS - -.. image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg +.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows :alt: Windows -.. image:: https://img.shields.io/badge/wheel-yes-brightgreen.svg - :target: https://pypi.org/pypi/Scrapy - :alt: Wheel Status - -.. image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg +.. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg :target: https://codecov.io/github/scrapy/scrapy?branch=master :alt: Coverage report -.. image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg +.. |conda| image:: https://anaconda.org/conda-forge/scrapy/badges/version.svg :target: https://anaconda.org/conda-forge/scrapy :alt: Conda Version +.. |deepwiki| image:: https://deepwiki.com/badge.svg + :target: https://deepwiki.com/scrapy/scrapy + :alt: Ask DeepWiki -Overview -======== - -Scrapy is a BSD-licensed fast high-level web crawling and web scraping framework, used to -crawl websites and extract structured data from their pages. It can be used for -a wide range of purposes, from data mining to monitoring and automated testing. - -Scrapy is maintained by Zyte_ (formerly Scrapinghub) and `many other -contributors`_. +Scrapy_ is a web scraping framework to extract structured data from websites. +It is cross-platform, and requires Python 3.10+. It is maintained by Zyte_ +(formerly Scrapinghub) and `many other contributors`_. .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors +.. _Scrapy: https://scrapy.org/ .. _Zyte: https://www.zyte.com/ -Check the Scrapy homepage at https://scrapy.org for more information, -including a list of features. - - -Requirements -============ - -* Python 3.9+ -* Works on Linux, Windows, macOS, BSD - -Install -======= - -The quick way: +Install with: .. code:: bash pip install scrapy -See the install section in the documentation at -https://docs.scrapy.org/en/latest/intro/install.html for more details. +And follow the documentation_ to learn how to use it. -Documentation -============= +.. _documentation: https://docs.scrapy.org/en/latest/ -Documentation is available online at https://docs.scrapy.org/ and in the ``docs`` -directory. +If you wish to contribute, see Contributing_. -Releases -======== - -You can check https://docs.scrapy.org/en/latest/news.html for the release notes. - -Community (blog, twitter, mail list, IRC) -========================================= - -See https://scrapy.org/community/ for details. - -Contributing -============ - -See https://docs.scrapy.org/en/master/contributing.html for details. - -Code of Conduct ---------------- - -Please note that this project is released with a Contributor `Code of Conduct `_. - -By participating in this project you agree to abide by its terms. -Please report unacceptable behavior to opensource@zyte.com. - -Companies using Scrapy -====================== - -See https://scrapy.org/companies/ for a list. - -Commercial Support -================== - -See https://scrapy.org/support/ for details. +.. _Contributing: https://docs.scrapy.org/en/master/contributing.html diff --git a/SECURITY.md b/SECURITY.md index bc64dec7b..c1a5482c6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,8 +4,8 @@ | Version | Supported | | ------- | ------------------ | -| 2.12.x | :white_check_mark: | -| < 2.12.x | :x: | +| 2.17.x | :white_check_mark: | +| < 2.17.x | :x: | ## Reporting a Vulnerability diff --git a/artwork/README.rst b/artwork/README.rst deleted file mode 100644 index c1880ef6c..000000000 --- a/artwork/README.rst +++ /dev/null @@ -1,20 +0,0 @@ -============== -Scrapy artwork -============== - -This folder contains the Scrapy artwork resources such as logos and fonts. - -scrapy-logo.jpg ---------------- - -The main Scrapy logo, in JPEG format. - -qlassik.zip ------------ - -The font used for the Scrapy logo. Homepage: https://www.dafont.com/qlassik.font - -scrapy-blog.logo.xcf --------------------- - -The logo used in the Scrapy blog, in Gimp format. diff --git a/artwork/qlassik.zip b/artwork/qlassik.zip deleted file mode 100644 index 2885c06ef..000000000 Binary files a/artwork/qlassik.zip and /dev/null differ diff --git a/artwork/scrapy-blog-logo.xcf b/artwork/scrapy-blog-logo.xcf deleted file mode 100644 index 320102604..000000000 Binary files a/artwork/scrapy-blog-logo.xcf and /dev/null differ diff --git a/artwork/scrapy-logo.jpg b/artwork/scrapy-logo.jpg deleted file mode 100644 index 4315ef8e1..000000000 Binary files a/artwork/scrapy-logo.jpg and /dev/null differ diff --git a/conftest.py b/conftest.py index e9765962a..27c398792 100644 --- a/conftest.py +++ b/conftest.py @@ -1,10 +1,21 @@ +from __future__ import annotations + +import os +from importlib.util import find_spec from pathlib import Path +from typing import TYPE_CHECKING import pytest from twisted.web.http import H2_ENABLED -from scrapy.utils.reactor import install_reactor +from scrapy.utils.reactor import set_asyncio_event_loop_policy +from scrapy.utils.reactorless import install_reactor_import_hook from tests.keys import generate_keys +from tests.mockserver.http import MockServer +from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd + +if TYPE_CHECKING: + from collections.abc import Generator def _py_files(folder): @@ -12,15 +23,15 @@ def _py_files(folder): collect_ignore = [ - # not a test, but looks like a test - "scrapy/utils/testsite.py", - "tests/ftpserver.py", - "tests/mockserver.py", - "tests/pipelines.py", - "tests/spiders.py", - # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess + # may need extra deps + "docs/_ext", + # contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerProcessSubprocess + *_py_files("tests/AsyncCrawlerProcess"), + # contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerRunnerSubprocess + *_py_files("tests/AsyncCrawlerRunner"), + # contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), - # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess + # contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerRunnerSubprocess *_py_files("tests/CrawlerRunner"), ] @@ -40,85 +51,91 @@ if not H2_ENABLED: ) ) - -@pytest.fixture() -def chdir(tmpdir): - """Change to pytest-provided temporary directory""" - tmpdir.chdir() +if find_spec("httpx2") is None and find_spec("httpx") is None: + collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py") -def pytest_addoption(parser): +def pytest_addoption(parser, pluginmanager): + if pluginmanager.hasplugin("twisted"): + return + # add the full choice set so that pytest doesn't complain about invalid choices in some cases parser.addoption( "--reactor", - default="default", - choices=["default", "asyncio"], + default="none", + choices=["asyncio", "default", "none"], ) -@pytest.fixture(scope="class") -def reactor_pytest(request): - if not request.cls: - # doctests - return None - request.cls.reactor_pytest = request.config.getoption("--reactor") - return request.cls.reactor_pytest +@pytest.fixture(scope="session") +def mockserver() -> Generator[MockServer]: + with MockServer() as mockserver: + yield mockserver -@pytest.fixture(autouse=True) -def only_asyncio(request, reactor_pytest): - if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio": - pytest.skip("This test is only run with --reactor=asyncio") +@pytest.fixture # function scope because it modifies os.environ +def proxy_server( + request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch +) -> Generator[str]: + kind = request.param + proxy = MitmProxy(mode="socks5" if kind == "socks5" else None) + url = proxy.start() + if kind == "https": + url = url.replace("http://", "https://") + monkeypatch.setenv("http_proxy", url) + monkeypatch.setenv("https_proxy", url) - -@pytest.fixture(autouse=True) -def only_not_asyncio(request, reactor_pytest): - if ( - request.node.get_closest_marker("only_not_asyncio") - and reactor_pytest == "asyncio" - ): - pytest.skip("This test is only run without --reactor=asyncio") - - -@pytest.fixture(autouse=True) -def requires_uvloop(request): - if not request.node.get_closest_marker("requires_uvloop"): - return try: - import uvloop - - del uvloop - except ImportError: - pytest.skip("uvloop is not installed") + yield kind + finally: + proxy.stop() -@pytest.fixture(autouse=True) -def requires_botocore(request): - if not request.node.get_closest_marker("requires_botocore"): - return - try: - import botocore - - del botocore - except ImportError: - pytest.skip("botocore is not installed") - - -@pytest.fixture(autouse=True) -def requires_boto3(request): - if not request.node.get_closest_marker("requires_boto3"): - return - try: - import boto3 - - del boto3 - except ImportError: - pytest.skip("boto3 is not installed") +@pytest.fixture(scope="session") +def reactor_pytest(request) -> str: + return request.config.getoption("--reactor") def pytest_configure(config): if config.getoption("--reactor") == "asyncio": - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + # Needed on Windows to switch from proactor to selector for Twisted reactor compatibility. + # If we decide to run tests with both, we will need to add a new option and check it here. + set_asyncio_event_loop_policy() + elif config.getoption("--reactor") == "none": + install_reactor_import_hook() -# Generate localhost certificate files, needed by some tests -generate_keys() +def pytest_runtest_setup(item): + # Skip tests based on reactor markers + reactor = item.config.getoption("--reactor") + + if item.get_closest_marker("requires_reactor") and reactor == "none": + pytest.skip('This test is only run when the --reactor value is not "none"') + + if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}: + pytest.skip( + 'This test is only run when the --reactor value is "asyncio" (default) or "none"' + ) + + if item.get_closest_marker("only_not_asyncio") and reactor in {"asyncio", "none"}: + pytest.skip( + 'This test is only run when the --reactor value is not "asyncio" (default) or "none"' + ) + + # Skip tests requiring optional dependencies + optional_deps = [ + "uvloop", + "botocore", + "boto3", + ] + + for module in optional_deps: + if item.get_closest_marker(f"requires_{module}") and find_spec(module) is None: + pytest.skip(f"{module} is not installed") + + if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None: + pytest.skip("mitmdump is not available") + + +# Generate localhost certificate files, needed by some tests (but only once if xdist is used) +if "PYTEST_XDIST_WORKER" not in os.environ: + generate_keys() diff --git a/docs/Makefile b/docs/Makefile index 48401bac8..ed8809902 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,96 +1,20 @@ -# -# Makefile for Scrapy documentation [based on Python documentation Makefile] -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# Minimal makefile for Sphinx documentation # -# You can set these variables from the command line. -PYTHON = python -SPHINXOPTS = -PAPER = -SOURCES = -SHELL = /usr/bin/env bash - -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees \ - -D latex_elements.papersize=$(PAPER) \ - $(SPHINXOPTS) . build/$(BUILDER) $(SOURCES) - -.PHONY: help update build html htmlhelp clean +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = build +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " text to make plain text files" - @echo " changes to make an overview over all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " watch build HTML docs, open in browser and watch for changes" + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -build-dirs: - mkdir -p build/$(BUILDER) build/doctrees +.PHONY: help Makefile -build: build-dirs - sphinx-build $(ALLSPHINXOPTS) - @echo - -build-ignore-errors: build-dirs - -sphinx-build $(ALLSPHINXOPTS) - @echo - - -html: BUILDER = html -html: build - @echo "Build finished. The HTML pages are in build/html." - -htmlhelp: BUILDER = htmlhelp -htmlhelp: build - @echo "Build finished; now you can run HTML Help Workshop with the" \ - "build/htmlhelp/pydoc.hhp project file." - -latex: BUILDER = latex -latex: build - @echo "Build finished; the LaTeX files are in build/latex." - @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ - "run these through (pdf)latex." - -text: BUILDER = text -text: build - @echo "Build finished; the text files are in build/text." - -changes: BUILDER = changes -changes: build - @echo "The overview file is in build/changes." - -linkcheck: BUILDER = linkcheck -linkcheck: build - @echo "Link check complete; look for any errors in the above output " \ - "or in build/$(BUILDER)/output.txt" - -linkfix: BUILDER = linkcheck -linkfix: build-ignore-errors - $(PYTHON) utils/linkfix.py - @echo "Fixing redirecting links in docs has finished; check all " \ - "replacements before committing them" - -doctest: BUILDER = doctest -doctest: build - @echo "Testing of doctests in the sources finished, look at the " \ - "results in build/doctest/output.txt" - -pydoc-topics: BUILDER = pydoc-topics -pydoc-topics: build - @echo "Building finished; now copy build/pydoc-topics/pydoc_topics.py " \ - "into the Lib/ directory" - -coverage: BUILDER = coverage -coverage: build - -htmlview: html - $(PYTHON) -c "import webbrowser; from pathlib import Path; \ - webbrowser.open(Path('build/html/index.html').resolve().as_uri())" - -clean: - -rm -rf build/* - -watch: htmlview - watchmedo shell-command -p '*.rst' -c 'make html' -R -D +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/README.rst b/docs/README.rst deleted file mode 100644 index 36dd5aea4..000000000 --- a/docs/README.rst +++ /dev/null @@ -1,68 +0,0 @@ -:orphan: - -====================================== -Scrapy documentation quick start guide -====================================== - -This file provides a quick guide on how to compile the Scrapy documentation. - - -Setup the environment ---------------------- - -To compile the documentation you need Sphinx Python library. To install it -and all its dependencies run the following command from this dir - -:: - - pip install -r requirements.txt - - -Compile the documentation -------------------------- - -To compile the documentation (to classic HTML output) run the following command -from this dir:: - - make html - -Documentation will be generated (in HTML format) inside the ``build/html`` dir. - - -View the documentation ----------------------- - -To view the documentation run the following command:: - - make htmlview - -This command will fire up your default browser and open the main page of your -(previously generated) HTML documentation. - - -Start over ----------- - -To clean up all generated documentation files and start from scratch run:: - - make clean - -Keep in mind that this command won't touch any documentation source files. - - -Recreating documentation on the fly ------------------------------------ - -There is a way to recreate the doc automatically when you make changes, you -need to install watchdog (``pip install watchdog``) and then use:: - - make watch - -Alternative method using tox ----------------------------- - -To compile the documentation to HTML run the following command:: - - tox -e docs - -Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir. diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py index 9b63f39f6..edb91bfb9 100644 --- a/docs/_ext/scrapydocs.py +++ b/docs/_ext/scrapydocs.py @@ -1,63 +1,67 @@ # pylint: disable=import-error +from collections.abc import Sequence from operator import itemgetter +from typing import Any, TypedDict from docutils import nodes +from docutils.nodes import Element, General, Node, document from docutils.parsers.rst import Directive -from docutils.parsers.rst.roles import set_classes +from sphinx.application import Sphinx from sphinx.util.nodes import make_refnode -class settingslist_node(nodes.General, nodes.Element): +class SettingData(TypedDict): + docname: str + setting_name: str + refid: str + + +class SettingslistNode(General, Element): pass class SettingsListDirective(Directive): - def run(self): - return [settingslist_node("")] + def run(self) -> Sequence[Node]: + return [SettingslistNode()] -def is_setting_index(node): - if node.tagname == "index" and node["entries"]: +def is_setting_index(node: Node) -> bool: + if node.tagname == "index" and node["entries"]: # type: ignore[index,attr-defined] # index entries for setting directives look like: # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')] - entry_type, info, refid = node["entries"][0][:3] + entry_type, info, _ = node["entries"][0][:3] # type: ignore[index] return entry_type == "pair" and info.endswith("; setting") return False -def get_setting_target(node): - # target nodes are placed next to the node in the doc tree - return node.parent[node.parent.index(node) + 1] - - -def get_setting_name_and_refid(node): +def get_setting_name_and_refid(node: Node) -> tuple[str, str]: """Extract setting name from directive index node""" - entry_type, info, refid = node["entries"][0][:3] + _, info, refid = node["entries"][0][:3] # type: ignore[index] return info.replace("; setting", ""), refid -def collect_scrapy_settings_refs(app, doctree): +def collect_scrapy_settings_refs(app: Sphinx, doctree: document) -> None: env = app.builder.env if not hasattr(env, "scrapy_all_settings"): - env.scrapy_all_settings = [] - - for node in doctree.traverse(is_setting_index): - targetnode = get_setting_target(node) - assert isinstance(targetnode, nodes.target), "Next node is not a target" + emptyList: list[SettingData] = [] + env.scrapy_all_settings = emptyList # type: ignore[attr-defined] + for node in doctree.findall(is_setting_index): setting_name, refid = get_setting_name_and_refid(node) - env.scrapy_all_settings.append( - { - "docname": env.docname, - "setting_name": setting_name, - "refid": refid, - } + env.scrapy_all_settings.append( # type: ignore[attr-defined] + SettingData( + docname=env.docname, + setting_name=setting_name, + refid=refid, + ) ) -def make_setting_element(setting_data, app, fromdocname): +def make_setting_element( + setting_data: SettingData, app: Sphinx, fromdocname: str +) -> Any: refnode = make_refnode( app.builder, fromdocname, @@ -73,77 +77,106 @@ def make_setting_element(setting_data, app, fromdocname): return item -def replace_settingslist_nodes(app, doctree, fromdocname): +def make_setting_markdown_item( + setting_data: SettingData, app: Sphinx, fromdocname: str +) -> str: + uri = app.builder.get_relative_uri(fromdocname, setting_data["docname"]) + if uri.startswith("#"): + target = f"#{setting_data['refid']}" + else: + target = f"{uri}#{setting_data['refid']}" + return f"* [{setting_data['setting_name']}]({target})" + + +def _iter_sorted_settings(env: Any, fromdocname: str) -> list[SettingData]: + return [ + d + for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined] + if fromdocname != d["docname"] + ] + + +def replace_settingslist_nodes( + app: Sphinx, doctree: document, fromdocname: str +) -> None: env = app.builder.env - for node in doctree.traverse(settingslist_node): + for node in doctree.findall(SettingslistNode): settings_list = nodes.bullet_list() settings_list.extend( [ make_setting_element(d, app, fromdocname) - for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) - if fromdocname != d["docname"] + for d in _iter_sorted_settings(env, fromdocname) ] ) node.replace_self(settings_list) -def setup(app): - app.add_crossref_type( - directivename="setting", - rolename="setting", - indextemplate="pair: %s; setting", - ) - app.add_crossref_type( - directivename="signal", - rolename="signal", - indextemplate="pair: %s; signal", - ) - app.add_crossref_type( - directivename="command", - rolename="command", - indextemplate="pair: %s; command", - ) - app.add_crossref_type( - directivename="reqmeta", - rolename="reqmeta", - indextemplate="pair: %s; reqmeta", - ) +def visit_settingslist_node_markdown(translator: Any, _node: Node) -> None: + builder = translator.builder + env = builder.env + fromdocname = getattr(builder, "current_doc_name", env.docname) + lines = [ + make_setting_markdown_item(setting_data, builder.app, fromdocname) + for setting_data in _iter_sorted_settings(env, fromdocname) + ] + if lines: + translator.add("\n".join(lines), prefix_eol=2, suffix_eol=2) + raise nodes.SkipNode + + +def depart_settingslist_node_markdown(_translator: Any, _node: Node) -> None: + return None + + +def source_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/blob/master/" + text + node = nodes.reference(rawtext, text, refuri=ref, **options) + return [node], [] + + +def issue_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/issues/" + text + node = nodes.reference(rawtext, "issue " + text, refuri=ref) + return [node], [] + + +def commit_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "https://github.com/scrapy/scrapy/commit/" + text + node = nodes.reference(rawtext, "commit " + text, refuri=ref) + return [node], [] + + +def rev_role( + name, rawtext, text: str, lineno, inliner, options=None, content=None +) -> tuple[list[Any], list[Any]]: + ref = "http://hg.scrapy.org/scrapy/changeset/" + text + node = nodes.reference(rawtext, "r" + text, refuri=ref) + return [node], [] + + +def setup(app: Sphinx) -> dict[str, Any]: app.add_role("source", source_role) app.add_role("commit", commit_role) app.add_role("issue", issue_role) app.add_role("rev", rev_role) - app.add_node(settingslist_node) + app.add_node( + SettingslistNode, + markdown=(visit_settingslist_node_markdown, depart_settingslist_node_markdown), + singlemarkdown=( + visit_settingslist_node_markdown, + depart_settingslist_node_markdown, + ), + ) app.add_directive("settingslist", SettingsListDirective) app.connect("doctree-read", collect_scrapy_settings_refs) app.connect("doctree-resolved", replace_settingslist_nodes) - - -def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/blob/master/" + text - set_classes(options) - node = nodes.reference(rawtext, text, refuri=ref, **options) - return [node], [] - - -def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/issues/" + text - set_classes(options) - node = nodes.reference(rawtext, "issue " + text, refuri=ref, **options) - return [node], [] - - -def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "https://github.com/scrapy/scrapy/commit/" + text - set_classes(options) - node = nodes.reference(rawtext, "commit " + text, refuri=ref, **options) - return [node], [] - - -def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]): - ref = "http://hg.scrapy.org/scrapy/changeset/" + text - set_classes(options) - node = nodes.reference(rawtext, "r" + text, refuri=ref, **options) - return [node], [] + return {"parallel_read_safe": True} diff --git a/docs/_ext/scrapyfixautodoc.py b/docs/_ext/scrapyfixautodoc.py new file mode 100644 index 000000000..e342e92cf --- /dev/null +++ b/docs/_ext/scrapyfixautodoc.py @@ -0,0 +1,21 @@ +""" +Must be included after 'sphinx.ext.autodoc'. Fixes unwanted 'alias of' behavior. +https://github.com/sphinx-doc/sphinx/issues/4422 +""" + +from typing import Any + +# pylint: disable=import-error +from sphinx.application import Sphinx + + +def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) -> bool: + if not skip: + # autodoc was generating the text "alias of" for the following members + return name in {"default_item_class", "default_selector_class"} + return skip + + +def setup(app: Sphinx) -> dict[str, Any]: + app.connect("autodoc-skip-member", maybe_skip_member) + return {"parallel_read_safe": True} diff --git a/docs/_static/custom.css b/docs/_static/custom.css index 64f16939c..1c2859deb 100644 --- a/docs/_static/custom.css +++ b/docs/_static/custom.css @@ -7,4 +7,50 @@ } .rst-content dl p + ol, .rst-content dl p + ul { margin-top: -6px; /* Compensates margin-top: 12px of p */ -} \ No newline at end of file +} + +/*override some styles in +sphinx-rtd-dark-mode/static/dark_mode_css/general.css*/ +.theme-switcher { + right: 0.4em !important; + top: 0.6em !important; + -webkit-box-shadow: 0px 3px 14px 4px rgba(0, 0, 0, 0.30) !important; + box-shadow: 0px 3px 14px 4px rgba(0, 0, 0, 0.30) !important; + height: 2em !important; + width: 2em !important; +} + +/*place the toggle button for dark mode +at the bottom right corner on small screens*/ +@media (max-width: 768px) { + .theme-switcher { + right: 0.4em !important; + bottom: 2.6em !important; + top: auto !important; + } +} + +/*persist blue color at the top left used in +default rtd theme*/ +html[data-theme="dark"] .wy-side-nav-search, +html[data-theme="dark"] .wy-nav-top { + background-color: #1d577d !important; +} + +/*all the styles below used to present +API objects nicely in dark mode*/ +html[data-theme="dark"] .sig.sig-object { + border-left-color: #3e4446 !important; + background-color: #202325 !important +} + +html[data-theme="dark"] .sig-name, +html[data-theme="dark"] .sig-prename, +html[data-theme="dark"] .property, +html[data-theme="dark"] .sig-param, +html[data-theme="dark"] .sig-paren, +html[data-theme="dark"] .sig-return-icon, +html[data-theme="dark"] .sig-return-typehint, +html[data-theme="dark"] .optional { + color: #e8e6e3 !important +} diff --git a/docs/_static/logo.svg b/docs/_static/logo.svg new file mode 100644 index 000000000..04b2d18a7 --- /dev/null +++ b/docs/_static/logo.svg @@ -0,0 +1 @@ + diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html new file mode 100644 index 000000000..29394799b --- /dev/null +++ b/docs/_templates/layout.html @@ -0,0 +1,23 @@ +{% extends "!layout.html" %} + +{# Overridden to include a link to scrapy.org, not just to the docs root #} +{%- block sidebartitle %} + +{# the logo helper function was removed in Sphinx 6 and deprecated since Sphinx 4 #} +{# the master_doc variable was renamed to root_doc in Sphinx 4 (master_doc still exists in later Sphinx versions) #} +{%- set _logo_url = logo_url|default(pathto('_static/' + (logo or ""), 1)) %} +{%- set _root_doc = root_doc|default(master_doc) %} +scrapy.org / docs + +{%- if READTHEDOCS or DEBUG %} + {%- if theme_version_selector or theme_language_selector %} +
+
+
+
+ {%- endif %} +{%- endif %} + +{%- include "searchbox.html" %} + +{%- endblock %} diff --git a/docs/_tests/quotes.html b/docs/_tests/quotes.html index f4002ecd1..d1cfd9020 100644 --- a/docs/_tests/quotes.html +++ b/docs/_tests/quotes.html @@ -16,13 +16,13 @@

- + Login - +

- +
@@ -34,16 +34,16 @@
Tags: - - + + change - + deep-thoughts - + thinking - + world - +
@@ -54,12 +54,12 @@
Tags: - - + + abilities - + choices - +
@@ -70,18 +70,18 @@
Tags: - - + + inspirational - + life - + live - + miracle - + miracles - +
@@ -92,16 +92,16 @@
Tags: - - + + aliteracy - + books - + classic - + humor - +
@@ -112,12 +112,12 @@
Tags: - - + + be-yourself - + inspirational - +
@@ -128,14 +128,14 @@
Tags: - - + + adulthood - + success - + value - +
@@ -146,12 +146,12 @@
Tags: - - + + life - + love - +
@@ -162,16 +162,16 @@
Tags: - - + + edison - + failure - + inspirational - + paraphrased - +
@@ -182,10 +182,10 @@
Tags: - - + + misattributed-eleanor-roosevelt - +
@@ -196,73 +196,73 @@
Tags: - - + + humor - + obvious - + simile - +
- +

Top Ten tags

- + love - + inspirational - + life - + humor - + books - + reading - + friendship - + friends - + truth - + simile - - + +
diff --git a/docs/_tests/quotes1.html b/docs/_tests/quotes1.html index f4002ecd1..d1cfd9020 100644 --- a/docs/_tests/quotes1.html +++ b/docs/_tests/quotes1.html @@ -16,13 +16,13 @@

- + Login - +

- +
@@ -34,16 +34,16 @@
Tags: - - + + change - + deep-thoughts - + thinking - + world - +
@@ -54,12 +54,12 @@
Tags: - - + + abilities - + choices - +
@@ -70,18 +70,18 @@
Tags: - - + + inspirational - + life - + live - + miracle - + miracles - +
@@ -92,16 +92,16 @@
Tags: - - + + aliteracy - + books - + classic - + humor - +
@@ -112,12 +112,12 @@
Tags: - - + + be-yourself - + inspirational - +
@@ -128,14 +128,14 @@
Tags: - - + + adulthood - + success - + value - +
@@ -146,12 +146,12 @@
Tags: - - + + life - + love - +
@@ -162,16 +162,16 @@
Tags: - - + + edison - + failure - + inspirational - + paraphrased - +
@@ -182,10 +182,10 @@
Tags: - - + + misattributed-eleanor-roosevelt - +
@@ -196,73 +196,73 @@
Tags: - - + + humor - + obvious - + simile - +
- +

Top Ten tags

- + love - + inspirational - + life - + humor - + books - + reading - + friendship - + friends - + truth - + simile - - + +
diff --git a/docs/conf.py b/docs/conf.py index fd8165db3..de722baac 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,17 +1,11 @@ -# Scrapy documentation build configuration file, created by -# sphinx-quickstart on Mon Nov 24 12:02:52 2008. +# Configuration file for the Sphinx documentation builder. # -# This file is execfile()d with the current directory set to its containing dir. -# -# The contents of this file are pickled, so don't put values in the namespace -# that aren't pickleable (module imports are okay, they're removed automatically). -# -# All configuration values have a default; values that are commented out -# serve to show the default. +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html -# pylint: disable=import-error import os import sys +from collections.abc import Sequence from pathlib import Path # If your extensions are in another directory, add it here. If the directory @@ -20,36 +14,28 @@ sys.path.append(str(Path(__file__).parent / "_ext")) sys.path.insert(0, str(Path(__file__).parent.parent)) -# General configuration -# --------------------- +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "Scrapy" +project_copyright = "Scrapy developers" +author = "Scrapy developers" + + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - "hoverxref.extension", "notfound.extension", "scrapydocs", - "sphinx.ext.autodoc", + "sphinx_scrapy", + "scrapyfixautodoc", # Must be after "sphinx.ext.autodoc" "sphinx.ext.coverage", - "sphinx.ext.intersphinx", - "sphinx.ext.viewcode", + "sphinx_rtd_dark_mode", ] -# Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] - -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8' - -# The master toctree document. -master_doc = "index" - -# General information about the project. -project = "Scrapy" -copyright = "Scrapy developers" +exclude_patterns = ["build", "Thumbs.db", ".DS_Store"] # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -65,140 +51,34 @@ except ImportError: version = "" release = "" -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -language = "en" - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -# today = '' -# Else, today_fmt is used as the format for a strftime call. -# today_fmt = '%B %d, %Y' - -# List of documents that shouldn't be included in the build. -# unused_docs = [] - -exclude_patterns = ["build"] - -# List of directories, relative to source directory, that shouldn't be searched -# for source files. -exclude_trees = [".build"] - -# The reST default role (used for this markup: `text`) to use for all documents. -# default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -# add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -# add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -# show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = "sphinx" - -# List of Sphinx warnings that will not be raised suppress_warnings = ["epub.unknown_project_files"] -# Options for HTML output -# ----------------------- +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. html_theme = "sphinx_rtd_theme" - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -# html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -# Add path to the RTD explicitly to robustify builds (otherwise might -# fail in a clean Debian build env) -import sphinx_rtd_theme - -html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - -# The style sheet to use for HTML and HTML Help pages. A file of that name -# must exist either in Sphinx' static/ path, or in one of the custom paths -# given in html_static_path. -# html_style = 'scrapydoc.css' - -# The name for this set of Sphinx documents. If None, it defaults to -# " v documentation". -# html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -# html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -# html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -# html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. html_last_updated_fmt = "%b %d, %Y" -# Custom sidebar templates, maps document names to template names. -# html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -# html_additional_pages = {} - -# If false, no module index is generated. -# html_use_modindex = True - -# If false, no index is generated. -# html_use_index = True - -# If true, the index is split into individual pages for each letter. -# html_split_index = False - -# If true, the reST sources are included in the HTML build as _sources/. -html_copy_source = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -# html_use_opensearch = '' - -# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). -# html_file_suffix = '' - -# Output file base name for HTML help builder. -htmlhelp_basename = "Scrapydoc" - html_css_files = [ "custom.css", ] +html_context = { + "display_github": True, + "github_user": "scrapy", + "github_repo": "scrapy", + "github_version": "master", + "conf_py_path": "/docs/", +} + # Set canonical URL from the Read the Docs Domain html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "") -# Options for LaTeX output -# ------------------------ - -# The paper size ('letter' or 'a4'). -# latex_paper_size = 'letter' - -# The font size ('10pt', '11pt' or '12pt'). -# latex_font_size = '10pt' +# -- Options for LaTeX output ------------------------------------------------ +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-latex-output # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). @@ -206,39 +86,22 @@ latex_documents = [ ("index", "Scrapy.tex", "Scrapy Documentation", "Scrapy developers", "manual"), ] -# The name of an image file (relative to this directory) to place at the top of -# the title page. -# latex_logo = None -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -# latex_use_parts = False +# -- Options for the linkcheck builder --------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-the-linkcheck-builder -# Additional stuff for the LaTeX preamble. -# latex_preamble = '' - -# Documents to append as an appendix to all manuals. -# latex_appendices = [] - -# If false, no module index is generated. -# latex_use_modindex = True - - -# Options for the linkcheck builder -# --------------------------------- - -# A list of regular expressions that match URIs that should not be checked when -# doing a linkcheck build. linkcheck_ignore = [ r"http://localhost:\d+", "http://hg.scrapy.org", - "http://directory.google.com/", + r"https://github.com/scrapy/scrapy/commit/\w+", r"https://github.com/scrapy/scrapy/issues/\d+", ] +linkcheck_anchors_ignore_for_url = ["https://github.com/pyca/cryptography/issues/2692"] + +# -- Options for the Coverage extension -------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/coverage.html#configuration -# Options for the Coverage extension -# ---------------------------------- coverage_ignore_pyobjects = [ # Contract’s add_pre_hook and add_post_hook are not documented because # they should be transparent to contract developers, for whom pre_hook and @@ -258,6 +121,10 @@ coverage_ignore_pyobjects = [ # Base classes of downloader middlewares are implementation details that # are not meant for users. r"^scrapy\.downloadermiddlewares\.\w*?\.Base\w*?Middleware", + # The interface methods of duplicate request filtering classes are already + # covered in the interface documentation part of the DUPEFILTER_CLASS + # setting documentation. + r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_crawler|request_seen|open|close|log)$", # Private exception used by the command-line interface implementation. r"^scrapy\.exceptions\.UsageError", # Methods of BaseItemExporter subclasses are only documented in @@ -275,52 +142,30 @@ coverage_ignore_pyobjects = [ ] -# Options for the InterSphinx extension -# ------------------------------------- +# -- Options for the InterSphinx extension ----------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration -intersphinx_mapping = { - "attrs": ("https://www.attrs.org/en/stable/", None), - "coverage": ("https://coverage.readthedocs.io/en/latest", None), - "cryptography": ("https://cryptography.io/en/latest/", None), - "cssselect": ("https://cssselect.readthedocs.io/en/latest", None), - "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None), - "parsel": ("https://parsel.readthedocs.io/en/latest/", None), - "pytest": ("https://docs.pytest.org/en/latest", None), - "python": ("https://docs.python.org/3", None), - "sphinx": ("https://www.sphinx-doc.org/en/master", None), - "tox": ("https://tox.wiki/en/latest/", None), - "twisted": ("https://docs.twisted.org/en/stable/", None), - "twistedapi": ("https://docs.twisted.org/en/stable/api/", None), - "w3lib": ("https://w3lib.readthedocs.io/en/latest", None), -} -intersphinx_disabled_reftypes = [] +intersphinx_disabled_reftypes: Sequence[str] = [] +# sphinx-scrapy --------------------------------------------------------------- -# Options for sphinx-hoverxref options -# ------------------------------------ +scrapy_intersphinx_enable = [ + "attrs", + "coverage", + "cryptography", + "cssselect", + "form2request", + "itemloaders", + "parsel", + "pytest", + "pypug", + "scrapy-lint", + "sphinx", + "tox", + "twisted", + "twistedapi", + "w3lib", +] -hoverxref_auto_ref = True -hoverxref_role_types = { - "class": "tooltip", - "command": "tooltip", - "confval": "tooltip", - "hoverxref": "tooltip", - "mod": "tooltip", - "ref": "tooltip", - "reqmeta": "tooltip", - "setting": "tooltip", - "signal": "tooltip", -} -hoverxref_roles = ["command", "reqmeta", "setting", "signal"] - - -def setup(app): - app.connect("autodoc-skip-member", maybe_skip_member) - - -def maybe_skip_member(app, what, name, obj, skip, options): - if not skip: - # autodocs was generating a text "alias of" for the following members - # https://github.com/sphinx-doc/sphinx/issues/4422 - return name in {"default_item_class", "default_selector_class"} - return skip +# -- Other options ------------------------------------------------------------ +default_dark_mode = False diff --git a/docs/contributing.rst b/docs/contributing.rst index f5c1c74b8..6d2e08fe8 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -6,8 +6,13 @@ Contributing to Scrapy .. important:: - Double check that you are reading the most recent version of this document at - https://docs.scrapy.org/en/master/contributing.html + Double check that you are reading the most recent version of this document + at https://docs.scrapy.org/en/master/contributing.html + + By participating in this project you agree to abide by the terms of our + `Code of Conduct + `_. Please + report unacceptable behavior to opensource@zyte.com. There are many ways to contribute to Scrapy. Here are some of them: @@ -246,14 +251,14 @@ Coding style Please follow these coding conventions when writing code for inclusion in Scrapy: -* We use `black `_ for code formatting. +* We use `Ruff `_ for code formatting. There is a hook in the pre-commit config that will automatically format your code before every commit. You can also - run black manually with ``tox -e pre-commit``. + run Ruff manually with ``tox -e pre-commit``. * Don't put your name in the code you contribute; git provides enough metadata to identify author of the code. - See https://docs.github.com/en/get-started/getting-started-with-git/setting-your-username-in-git + See https://docs.github.com/en/get-started/git-basics/setting-your-username-in-git for setup instructions. .. _scrapy-pre-commit: @@ -318,9 +323,10 @@ deprecation removals are documented in the :ref:`release notes `. Tests ===== -Tests are implemented using the :doc:`Twisted unit-testing framework -`. Running tests requires -:doc:`tox `. +Tests are implemented using pytest_. Running tests requires :doc:`tox +`. + +.. _pytest: https://pytest.org .. _running-tests: @@ -366,6 +372,21 @@ To see coverage report install :doc:`coverage ` see output of ``coverage --help`` for more options like html or xml report. +Some tests need a ``mitmdump`` executable (from mitmproxy_) to test against a +fully featured proxy server; they are skipped when one cannot be found +(``mitmproxy`` is intentionally not a test dependency that would be installed +into test venvs, as that sometimes leads to various dependency conflicts). +To run these tests, make ``mitmdump`` available in one of these ways: + +* install ``mitmproxy`` so that ``mitmdump`` is on your ``PATH``, e.g. with + pipx_ (``pipx install mitmproxy``) or uv_ (``uv tool install mitmproxy``); + +* have uv_ installed, in which case the tests will run + ``uvx --from mitmproxy mitmdump``; + +* set the ``MITMDUMP`` environment variable to the path of a ``mitmdump`` + executable. + Writing tests ------------- @@ -385,8 +406,7 @@ And their unit-tests are in:: .. _issue tracker: https://github.com/scrapy/scrapy/issues .. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users -.. _Scrapy subreddit: https://reddit.com/r/scrapy -.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS +.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/ .. _tests/: https://github.com/scrapy/scrapy/tree/master/tests .. _open issues: https://github.com/scrapy/scrapy/issues .. _PEP 257: https://peps.python.org/pep-0257/ @@ -394,3 +414,6 @@ And their unit-tests are in:: .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 .. _test coverage: https://app.codecov.io/gh/scrapy/scrapy +.. _mitmproxy: https://mitmproxy.org/ +.. _pipx: https://pipx.pypa.io/ +.. _uv: https://docs.astral.sh/uv/ diff --git a/docs/faq.rst b/docs/faq.rst index f81ec3601..0446a6868 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -82,10 +82,18 @@ to steal from us! Does Scrapy work with HTTP proxies? ----------------------------------- -Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP -Proxy downloader middleware. See +Yes. Support for HTTP proxies is provided through the HTTP Proxy downloader +middleware. See :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`. +Does Scrapy work with SOCKS proxies? +------------------------------------ + +Yes, when using +:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. See +:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and the +handler documentation. + How can I scrape an item with attributes in different pages? ------------------------------------------------------------ @@ -96,30 +104,13 @@ How can I simulate a user login in my spider? See :ref:`topics-request-response-ref-request-userlogin`. + .. _faq-bfo-dfo: Does Scrapy crawl in breadth-first or depth-first order? -------------------------------------------------------- -By default, Scrapy uses a `LIFO`_ queue for storing pending requests, which -basically means that it crawls in `DFO order`_. This order is more convenient -in most cases. - -If you do want to crawl in true `BFO order`_, you can do it by -setting the following settings: - -.. code-block:: python - - DEPTH_PRIORITY = 1 - SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue" - SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue" - -While pending requests are below the configured values of -:setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP`, those requests are sent -concurrently. As a result, the first few requests of a crawl rarely follow the -desired order. Lowering those settings to ``1`` enforces the desired order, but -it significantly slows down the crawl as a whole. +:ref:`DFO by default, but other orders are possible `. My Scrapy crawler has memory leaks. What can I do? @@ -145,7 +136,7 @@ middleware with a :ref:`custom downloader middleware ` that requires less memory. For example: - If your domain names are similar enough, use your own regular expression - instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into + instead of joining the strings in :attr:`~scrapy.Spider.allowed_domains` into a complex regular expression. - If you can meet the installation requirements, use pyre2_ instead of @@ -294,7 +285,8 @@ consume a lot of memory. In order to avoid parsing all the entire feed at once in memory, you can use the :func:`~scrapy.utils.iterators.xmliter_lxml` and :func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what -:class:`~scrapy.spiders.XMLFeedSpider` uses. +:class:`~scrapy.spiders.XMLFeedSpider` and +:class:`~scrapy.spiders.CSVFeedSpider` use. .. autofunction:: scrapy.utils.iterators.xmliter_lxml @@ -340,8 +332,8 @@ section of the site (which varies each time). In that case, the credentials to log in would be settings, while the url of the section to scrape would be a spider argument. -I'm scraping a XML document and my XPath selector doesn't return any items --------------------------------------------------------------------------- +I'm scraping an XML document and my XPath selector doesn't return any items +--------------------------------------------------------------------------- You may need to remove namespaces. See :ref:`removing-namespaces`. @@ -361,21 +353,27 @@ method for this purpose. For example: from copy import deepcopy - from itemadapter import is_item, ItemAdapter + from itemadapter import ItemAdapter + from scrapy import Request class MultiplyItemsMiddleware: - def process_spider_output(self, response, result, spider): - for item in result: - if is_item(item): - adapter = ItemAdapter(item) - for _ in range(adapter["multiply_by"]): - yield deepcopy(item) + def process_spider_output(self, response, result): + for item_or_request in result: + if isinstance(item_or_request, Request): + yield item_or_request + continue + adapter = ItemAdapter(item_or_request) + for _ in range(adapter["multiply_by"]): + yield deepcopy(item_or_request) Does Scrapy support IPv6 addresses? ----------------------------------- -Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. +Yes, but when using +:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` or +:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` you need to +set :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. Note that by doing so, you lose the ability to set a specific timeout for DNS requests (the value of the :setting:`DNS_TIMEOUT` setting is ignored). @@ -386,8 +384,9 @@ How to deal with ``: filedescriptor out of range in select() ---------------------------------------------------------------------------------------------- This issue `has been reported`_ to appear when running broad crawls in macOS, where the default -Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a -different reactor is possible by using the :setting:`TWISTED_REACTOR` setting. +Twisted reactor was :class:`twisted.internet.selectreactor.SelectReactor` at that time. +If you have switched to this reactor using the :setting:`TWISTED_REACTOR` setting you can switch +to a different one in the same way. .. _faq-stop-response-download: @@ -410,9 +409,8 @@ How can I make a blank request? ------------------------------- .. code-block:: python - - from scrapy import Request + from scrapy import Request blank_request = Request("data:,") @@ -433,7 +431,3 @@ See :issue:`2680`. .. _has been reported: https://github.com/scrapy/scrapy/issues/2905 .. _Python standard library modules: https://docs.python.org/3/py-modindex.html .. _Python package: https://pypi.org/ -.. _user agents: https://en.wikipedia.org/wiki/User_agent -.. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type) -.. _DFO order: https://en.wikipedia.org/wiki/Depth-first_search -.. _BFO order: https://en.wikipedia.org/wiki/Breadth-first_search diff --git a/docs/index.rst b/docs/index.rst index 1a9cf636c..688cab81b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -24,7 +24,7 @@ Having trouble? We'd like to help! * Ask or search questions in `StackOverflow using the scrapy tag`_. * Ask or search questions in the `Scrapy subreddit`_. * Search for questions on the archives of the `scrapy-users mailing list`_. -* Ask a question in the `#scrapy IRC channel`_, +* Ask a question in the `#scrapy IRC channel`_. * Report bugs with Scrapy in our `issue tracker`_. * Join the Discord community `Scrapy Discord`_. @@ -91,15 +91,15 @@ Basic concepts :doc:`topics/selectors` Extract the data from web pages using XPath. -:doc:`topics/shell` - Test your extraction code in an interactive environment. - :doc:`topics/items` Define the data you want to scrape. :doc:`topics/loaders` Populate your items with the extracted data. +:doc:`topics/shell` + Test your extraction code in an interactive environment. + :doc:`topics/item-pipeline` Post-process and store your scraped data. @@ -128,18 +128,14 @@ Built-in services topics/logging topics/stats - topics/email topics/telnetconsole :doc:`topics/logging` - Learn how to use Python's builtin logging on Scrapy. + Learn how to use Python's built-in logging on Scrapy. :doc:`topics/stats` Collect statistics about your scraping crawler. -:doc:`topics/email` - Send email notifications when certain events occur. - :doc:`topics/telnetconsole` Inspect a running crawler using a built-in Python console. @@ -155,6 +151,7 @@ Solving specific problems topics/debug topics/contracts topics/practices + topics/security topics/broad-crawls topics/developer-tools topics/dynamic-content @@ -179,6 +176,10 @@ Solving specific problems :doc:`topics/practices` Get familiar with some Scrapy common practices. +:doc:`topics/security` + Understand the security implications of Scrapy defaults and how to harden + them. + :doc:`topics/broad-crawls` Tune Scrapy for crawling a lot domains in parallel. @@ -229,6 +230,7 @@ Extending Scrapy topics/signals topics/scheduler topics/exporters + topics/download-handlers topics/components topics/api @@ -257,6 +259,9 @@ Extending Scrapy :doc:`topics/exporters` Quickly export your scraped items to a file (XML, CSV, etc). +:doc:`topics/download-handlers` + Customize how requests are downloaded or add support for new URL schemes. + :doc:`topics/components` Learn the common API and some good practices when building custom Scrapy components. diff --git a/docs/intro/install.rst b/docs/intro/install.rst index 82a0e18c5..cba8c15a1 100644 --- a/docs/intro/install.rst +++ b/docs/intro/install.rst @@ -9,7 +9,7 @@ Installation guide Supported Python versions ========================= -Scrapy requires Python 3.9+, either the CPython implementation (default) or +Scrapy requires Python 3.10+, either the CPython implementation (default) or the PyPy implementation (see :ref:`python:implementations`). .. _intro-install-scrapy: @@ -89,6 +89,56 @@ just like any other Python package. (See :ref:`platform-specific guides ` below for non-Python dependencies that you may need to install beforehand). +.. _extras: + +Optional extras +=============== + +Scrapy provides optional :ref:`extras ` +that install additional dependencies to enable specific features. To install +Scrapy with one or more extras, list them in square brackets: + +.. code-block:: console + + pip install scrapy[s3,images] + +The following extras are available: + +.. list-table:: + :header-rows: 1 + + * - Extra + - Provides + * - ``bpython`` + - :ref:`bpython shell ` + * - ``brotli`` + - :ref:`Brotli response decompression ` + * - ``gcs`` + - :ref:`Google Cloud Storage ` for + :ref:`feed exports ` and + :ref:`media pipelines ` + * - ``httpx`` + - :ref:`httpx-handler`, including its HTTP/2 and SOCKS proxy support + * - ``images`` + - :ref:`Images pipeline ` + * - ``ipython`` + - :ref:`IPython shell ` + * - ``ptpython`` + - :ref:`ptpython shell ` + * - ``robotparser`` + - :ref:`Robotexclusionrulesparser robots.txt parsing ` + * - ``s3`` + - :ref:`Amazon S3 ` storage for + :ref:`feed exports `, + :ref:`media pipelines `, and + :ref:`S3 downloads ` + * - ``twisted-http2`` + - :ref:`twisted-http2-handler` + * - ``uvloop`` + - `uvloop `_ event loop + * - ``zstd`` + - :ref:`Zstandard response decompression ` + .. _intro-install-platform-notes: @@ -111,7 +161,7 @@ Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with:: To install Scrapy on Windows using ``pip``: .. warning:: - This installation method requires “Microsoft Visual C++” for installing some + This installation method requires “Microsoft Visual C++” for installing some Scrapy dependencies, which demands significantly more disk space than Anaconda. #. Download and execute `Microsoft C++ Build Tools`_ to install the Visual Studio Installer. @@ -123,7 +173,7 @@ To install Scrapy on Windows using ``pip``: #. Check the installation details and make sure following packages are selected as optional components: * **MSVC** (e.g MSVC v142 - VS 2019 C++ x64/x86 build tools (v14.23) ) - + * **Windows SDK** (e.g Windows 10 SDK (10.0.18362.0)) #. Install the Visual Studio Build Tools. @@ -230,8 +280,8 @@ Installing Scrapy with PyPy on Windows is not tested. You can check that Scrapy is installed correctly by running ``scrapy bench``. If this command gives errors such as ``TypeError: ... got 2 unexpected keyword arguments``, this means -that setuptools was unable to pick up one PyPy-specific dependency. -To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``. +that the ``PyPyDispatcher`` dependency wasn't installed. To fix this issue, run +``pip install 'PyPyDispatcher>=2.1.0'``. .. _intro-install-troubleshooting: @@ -263,7 +313,6 @@ reinstall Twisted with the :code:`tls` extra option:: For details, see `Issue #2473 `_. .. _Python: https://www.python.org/ -.. _pip: https://pip.pypa.io/en/latest/installing/ .. _lxml: https://lxml.de/index.html .. _parsel: https://pypi.org/project/parsel/ .. _w3lib: https://pypi.org/project/w3lib/ @@ -273,8 +322,7 @@ For details, see `Issue #2473 `_. .. _setuptools: https://pypi.org/pypi/setuptools .. _homebrew: https://brew.sh/ .. _zsh: https://www.zsh.org/ -.. _Anaconda: https://docs.anaconda.com/anaconda/ +.. _Anaconda: https://www.anaconda.com/docs/main .. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html -.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio .. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/ .. _conda-forge: https://conda-forge.org/ diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index d05e46551..5937c5860 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -83,16 +83,17 @@ While this enables you to do very fast crawls (sending multiple concurrent requests at the same time, in a fault-tolerant way) Scrapy also gives you control over the politeness of the crawl through :ref:`a few settings `. You can do things like setting a download delay between -each request, limiting the amount of concurrent requests per domain or per IP, and +each request, limiting the amount of concurrent requests per domain, and even :ref:`using an auto-throttling extension ` that tries to figure these settings out automatically. .. note:: This is using :ref:`feed exports ` to generate the - JSON file, you can easily change the export format (XML or CSV, for example) or the - storage backend (FTP or `Amazon S3`_, for example). You can also write an - :ref:`item pipeline ` to store the items in a database. + JSON Lines file, you can easily change the export format (XML or CSV, for + example) or the storage backend (FTP or `Amazon S3`_, for example). You can + also write an :ref:`item pipeline ` to store the + items in a database. .. _topics-whatelse: @@ -150,7 +151,7 @@ The next steps for you are to :ref:`install Scrapy `, a full-blown Scrapy project and `join the community`_. Thanks for your interest! -.. _join the community: https://scrapy.org/community/ +.. _join the community: https://www.scrapy.org/community .. _web scraping: https://en.wikipedia.org/wiki/Web_scraping .. _Amazon Associates Web Services: https://affiliate-program.amazon.com/welcome/ecs .. _Amazon S3: https://aws.amazon.com/s3/ diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 6e6caebf1..c4e04364b 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -94,7 +94,7 @@ This is the code for our first Spider. Save it in a file named class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def start(self): urls = [ "https://quotes.toscrape.com/page/1/", "https://quotes.toscrape.com/page/2/", @@ -116,10 +116,10 @@ and defines some attributes and methods: unique within a project, that is, you can't set the same name for different Spiders. -* :meth:`~scrapy.Spider.start_requests`: must return an iterable of - Requests (you can return a list of requests or write a generator function) - which the Spider will begin to crawl from. Subsequent requests will be - generated successively from these initial requests. +* :meth:`~scrapy.Spider.start`: must be an asynchronous generator that + yields requests (and, optionally, items) for the spider to start crawling. + Subsequent requests will be generated successively from these initial + requests. * :meth:`~scrapy.Spider.parse`: a method that will be called to handle the response downloaded for each of the requests made. The response parameter @@ -164,21 +164,22 @@ for the respective URLs, as our ``parse`` method instructs. What just happened under the hood? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Scrapy schedules the :class:`scrapy.Request ` objects -returned by the ``start_requests`` method of the Spider. Upon receiving a -response for each one, it instantiates :class:`~scrapy.http.Response` objects -and calls the callback method associated with the request (in this case, the -``parse`` method) passing the response as an argument. +Scrapy sends the first :class:`scrapy.Request ` objects yielded +by the :meth:`~scrapy.Spider.start` spider method. Upon receiving a +response for each one, Scrapy calls the callback method associated with the +request (in this case, the ``parse`` method) with a +:class:`~scrapy.http.Response` object. -A shortcut to the start_requests method ---------------------------------------- -Instead of implementing a :meth:`~scrapy.Spider.start_requests` method -that generates :class:`scrapy.Request ` objects from URLs, -you can just define a :attr:`~scrapy.Spider.start_urls` class attribute -with a list of URLs. This list will then be used by the default implementation -of :meth:`~scrapy.Spider.start_requests` to create the initial requests -for your spider. +A shortcut to the ``start`` method +---------------------------------- + +Instead of implementing a :meth:`~scrapy.Spider.start` method that yields +:class:`~scrapy.Request` objects from URLs, you can define a +:attr:`~scrapy.Spider.start_urls` class attribute with a list of URLs. This +list will then be used by the default implementation of +:meth:`~scrapy.Spider.start` to create the initial requests for your +spider. .. code-block:: python @@ -292,7 +293,7 @@ As an alternative, you could've written: >>> response.css("title::text")[0].get() 'Quotes to Scrape' -Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will +Accessing an index on a :class:`~scrapy.selector.SelectorList` instance will raise an :exc:`IndexError` exception if there are no results: .. code-block:: pycon @@ -302,8 +303,8 @@ raise an :exc:`IndexError` exception if there are no results: ... IndexError: list index out of range -You might want to use ``.get()`` directly on the -:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` +You might want to use ``.get()`` directly on the +:class:`~scrapy.selector.SelectorList` instance instead, which returns ``None`` if there are no results: .. code-block:: pycon @@ -794,7 +795,7 @@ with a specific tag, building the URL based on the argument: class QuotesSpider(scrapy.Spider): name = "quotes" - def start_requests(self): + async def start(self): url = "https://quotes.toscrape.com/" tag = getattr(self, "tag", None) if tag is not None: diff --git a/docs/news.rst b/docs/news.rst index 924abb7a1..8f8477eaa 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -3,6 +3,2508 @@ Release notes ============= +Scrapy VERSION (unreleased) +--------------------------- + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The following runtime usage of zope.interface_ interfaces is removed: + + - :class:`~scrapy.spiderloader.SpiderLoader` and + :class:`~scrapy.spiderloader.DummySpiderLoader` are no longer marked + as implementing the ``ISpiderLoader`` interface. + + - :func:`~scrapy.spiderloader.get_spider_loader` no longer checks that the + configured spider loader implements the ``ISpiderLoader`` interface. + + - :class:`~scrapy.extensions.feedexport.BlockingFeedStorage`, + :class:`~scrapy.extensions.feedexport.FileFeedStorage` and + :class:`~scrapy.extensions.feedexport.StdoutFeedStorage` are no longer + marked as implementing the ``IFeedStorage`` interface. + + (:issue:`6585`, :issue:`7731`) + +.. _release-2.17.0: + +Scrapy 2.17.0 (2026-07-07) +-------------------------- + +Highlights: + +- Security bug fixes + +- HTTP/2 and SOCKS proxy support for ``HttpxDownloadHandler`` + +- Improved settings for changing allowed TLS versions + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- ``s3://`` requests now use HTTPS by default, instead of plaintext HTTP. + + Previously, :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler` + sent signed S3 requests over plaintext HTTP unless + ``request.meta["is_secure"]`` was set to a true value, exposing the request + path, the AWS ``Authorization`` header, the ``X-Amz-Security-Token`` header + (when using temporary credentials), and the response contents to network + attackers, who could also tamper with responses. See the `76g3-c3x4-crvx`_ + security advisory for details. + + To restore the previous behavior for a given request, set + ``request.meta["is_secure"]`` to ``False``. + + .. _76g3-c3x4-crvx: https://github.com/scrapy/scrapy/security/advisories/GHSA-76g3-c3x4-crvx + +Deprecations +~~~~~~~~~~~~ + +- The ``DOWNLOADER_CLIENT_TLS_METHOD`` setting is deprecated. You should use + the :setting:`DOWNLOAD_TLS_MIN_VERSION` and/or + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings instead if you want to change + the TLS method selection. + (:issue:`3288`, :issue:`6546`) + +- The following spider attributes are deprecated in favor of settings: + + - ``http_user`` (use :setting:`HTTPAUTH_USER`) + + - ``http_pass`` (use :setting:`HTTPAUTH_PASS`) + + - ``http_auth_domain`` (use :setting:`HTTPAUTH_DOMAIN`) + + (:issue:`7590`) + +- The ``scrapy.commands.ScrapyCommand.help()`` method is deprecated. It was + never called by Scrapy. + (:issue:`7626`, :issue:`7633`) + +- The following TLS-related functions and constants, intended for internal + use, are deprecated: + + - ``scrapy.core.downloader.tls.METHOD_TLS`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv10`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv11`` + + - ``scrapy.core.downloader.tls.METHOD_TLSv12`` + + - ``scrapy.core.downloader.tls.openssl_methods`` + + - ``scrapy.core.downloader.tls.DEFAULT_CIPHERS`` + + - ``scrapy.utils.ssl.ffi_buf_to_string()`` + + - ``scrapy.utils.ssl.get_temp_key_info()`` + + - ``scrapy.utils.ssl.x509name_to_string()`` + + (:issue:`6546`, :issue:`7619`, :issue:`7665`) + +- The ``CRAWLSPIDER_FOLLOW_LINKS`` setting is deprecated. You can set + ``follow=False`` in your rules to achieve the same effect. + (:issue:`7592`) + +- Instantiating + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + without a ``crawler`` argument is deprecated. + (:issue:`7655`) + +- Instantiating + :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` without a + ``settings`` argument is deprecated. + (:issue:`7664`) + +New features +~~~~~~~~~~~~ + +- Added support for HTTP/2 requests to + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. It + requires setting the new :setting:`HTTPX_HTTP2_ENABLED` setting to + ``True``. + (:issue:`7575`) + +- Added support for SOCKS proxies to + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. + (:issue:`747`, :issue:`7575`) + +- Added :setting:`DOWNLOAD_TLS_MIN_VERSION` and + :setting:`DOWNLOAD_TLS_MAX_VERSION` settings as replacements for the + ``DOWNLOADER_CLIENT_TLS_METHOD`` setting (which is now deprecated). + Compared to the old setting, they support specifying a range of allowed + versions and support newer TLS versions. + (:issue:`4821`, :issue:`6546`) + +- Added :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS` and + :setting:`HTTPAUTH_DOMAIN` settings and :reqmeta:`http_user`, + :reqmeta:`http_pass` and :reqmeta:`http_auth_domain` meta keys as more + flexible ways to set HTTP authentication data. + (:issue:`7590`) + +- Added a :reqmeta:`verbatim_url` meta key that can be set to ``True`` to + skip request URL canonicalization. + (:issue:`7473`) + +- Added ``deny_tags`` and ``deny_attrs`` arguments to :class:`LinkExtractor + `. + (:issue:`6321`, :issue:`7679`) + +- :attr:`scrapy.Item.fields` now returns the fields in the definition order + instead of the alphabetical one. + (:issue:`7015`, :issue:`7694`) + +- Added a :setting:`RETRY_GIVE_UP_LOG_LEVEL` setting, a + :reqmeta:`give_up_log_level` meta key and a ``give_up_log_level`` argument + of the + :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function that + allow changing the log level of the message logged when the retry limit has + been reached. + (:issue:`4622`, :issue:`5297`, :issue:`7567`) + +- It's now possible to set :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` to + ``None`` to use the default ciphers of the underlying TLS implementation. + (:issue:`7499`, :issue:`7665`) + +Improvements +~~~~~~~~~~~~ + +- :class:`~scrapy.FormRequest` is no longer deprecated, only its + ``from_response()`` method is still deprecated. + (:issue:`7561`, :issue:`7671`) + +- Switched the item definition in the default project template from a + :class:`scrapy.item.Item` to a dataclass. + (:issue:`7493`, :issue:`7513`) + +- Fixed deprecation warnings with pyOpenSSL 26.3.0. + (:issue:`7619`) + +- Removed the runtime warnings for :attr:`Spider.allowed_domains + ` containing URLs or domains with ports + instead of just domains and for spider classes having a ``start_url`` + attribute instead of :class:`~scrapy.spiders.Spider.start_urls`. Please use + :doc:`scrapy-lint ` to find mistakes in your spider code + instead. + (:issue:`4421`, :issue:`7627`) + +- :func:`scrapy.utils.test.get_crawler` now disables + :setting:`TELNETCONSOLE_ENABLED` by default. + (:issue:`7644`) + +- Other code refactoring and improvements. + (:issue:`7409`, :issue:`7593`, :issue:`7594`, :issue:`7611`, :issue:`7649`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` no + longer ignores proxy credentials for redirected or retried requests. + (:issue:`7601`, :issue:`7630`) + +- :class:`~scrapy.extensions.feedexport.GCSFeedStorage` now closes the + temporary file after the upload. + (:issue:`7546`) + +- Fixed ``scrapy shell `` running a full spider crawl when there is a + spider for the requested URL. This bug was introduced in Scrapy 2.13.0. + (:issue:`7552`, :issue:`7557`) + +- The :setting:`IMAGES_STORE_S3_ACL` and :setting:`IMAGES_STORE_GCS_ACL` + settings are no longer ignored. This bug was introduced in Scrapy 2.12.0. + (:issue:`7597`, :issue:`7614`) + +- :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` now closes + the connection after making the request. + (:issue:`7602`, :issue:`7667`) + +- Removed the deprecated ``spider`` argument from the pipeline defined in the + default project template. + (:issue:`7676`) + +- Fixed ``scrapy genspider --edit`` not working. + (:issue:`7260`, :issue:`7683`) + +- When a :class:`~scrapy.crawler.Crawler` instance is passed to + :meth:`AsyncCrawlerRunner.create_crawler() + ` or + :meth:`CrawlerRunner.create_crawler() + `, settings from both classes + are now merged, previously only the settings from the + :class:`~scrapy.crawler.Crawler` instance were used. + (:issue:`1280`, :issue:`7647`) + +- Fixed several issues with cookie handling in + :func:`scrapy.utils.request.request_to_curl`. + (:issue:`7603`, :issue:`7675`, :issue:`7684`) + +- Fixed :class:`scrapy.resolver.CachingThreadedResolver` not disabling the + cache when :setting:`DNSCACHE_ENABLED` is set to ``False``. + (:issue:`7663`) + +- Fixed :func:`scrapy.utils.response.open_in_browser` not removing comments + when looking for the ```` tag. + (:issue:`7506`) + +- Fixed checking for deprecated methods in custom :setting:`ITEM_PROCESSOR` + implementations. + (:issue:`7589`) + +- Fixed :func:`scrapy.utils.url.strip_url` corrupting some URLs with + credentials. + (:issue:`7604`, :issue:`7605`) + +- :func:`scrapy.utils.misc.rel_has_nofollow` now ignores the case when + looking for "nofollow" strings. + (:issue:`7632`) + +- Fixed an exception in :class:`scrapy.utils.sitemap.Sitemap` when parsing + some malformed sitemaps. + (:issue:`7686`, :issue:`7687`) + +Documentation +~~~~~~~~~~~~~ + +- Mentioned :doc:`scrapy-lint ` in the docs. + (:issue:`4421`, :issue:`7627`) + +- Added the docs about :ref:`security considerations `. + (:issue:`7389`, :issue:`7678`) + +- Improved the :ref:`item pipeline docs `. + (:issue:`2350`, :issue:`7676`) + +- Documented which stats are collected by + :class:`~scrapy.extensions.corestats.CoreStats`. + (:issue:`7421`) + +- Switched documentation examples from using :class:`scrapy.item.Item` to + using dataclasses. + (:issue:`7493`, :issue:`7513`) + +- Added feature comparison tables to the :ref:`download handler + ` docs. + (:issue:`7575`) + +- Improved the docs for :ref:`logging settings `. + (:issue:`6909`, :issue:`7668`) + +- Documented a way to :ref:`improve startup time and memory usage + ` by using :setting:`SPIDER_MODULES`. + (:issue:`7576`, :issue:`7600`) + +- Clarified handling of the ``type`` argument of :class:`~scrapy.Selector`. + (:issue:`7704`) + +- Other documentation improvements and fixes. + (:issue:`4954`, + :issue:`6120`, + :issue:`7286`, + :issue:`7564`, + :issue:`7573`, + :issue:`7598`, + :issue:`7599`, + :issue:`7698`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Fixed deprecation warnings with pytest 9.1.0. + (:issue:`7621`) + +- Type hints improvements and fixes. + (:issue:`6958`, :issue:`7586`) + +- CI and test improvements and fixes. + (:issue:`5954`, + :issue:`7002`, + :issue:`7017`, + :issue:`7247`, + :issue:`7508`, + :issue:`7545`, + :issue:`7566`, + :issue:`7574`, + :issue:`7585`, + :issue:`7595`, + :issue:`7608`, + :issue:`7610`, + :issue:`7612`, + :issue:`7616`, + :issue:`7625`, + :issue:`7637`, + :issue:`7639`, + :issue:`7640`, + :issue:`7641`, + :issue:`7642`, + :issue:`7643`, + :issue:`7644`, + :issue:`7645`, + :issue:`7646`, + :issue:`7654`, + :issue:`7655`, + :issue:`7664`, + :issue:`7672`, + :issue:`7677`, + :issue:`7680`, + :issue:`7682`, + :issue:`7692`) + +.. _release-2.16.0: + +Scrapy 2.16.0 (2026-05-19) +-------------------------- + +Highlights: + +- Official support for Python 3.14 + +- Support for Twisted 26.4.0+ + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Increased the minimum versions of the following dependencies: + + - service_identity_: 18.1.0 → 23.1.0 + + (:issue:`7347`) + +- Added support for Twisted 26.4.0+. + (:issue:`7347`, :issue:`7505`, :issue:`7520`) + +- Added support for Python 3.14. + (:issue:`6604`, :issue:`7460`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The following classes and functions, intended for internal use by + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + and :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`, have + been made private: + + - ``scrapy.core.downloader.handlers.http11.ScrapyAgent`` + + - ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingAgent`` + + - ``scrapy.core.downloader.handlers.http11.TunnelingTCP4ClientEndpoint`` + + - ``scrapy.core.downloader.handlers.http11.tunnel_request_data()`` + + - ``scrapy.core.downloader.handlers.http2.ScrapyH2Agent`` + + (:issue:`7496`, :issue:`7510`) + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.FormRequest`` is deprecated. You can use the :doc:`form2request + ` library instead, see :ref:`form`. + (:issue:`6438`) + +- ``scrapy.utils.python.MutableChain`` is deprecated. + (:issue:`7504`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The ``start_requests()`` method of :class:`~scrapy.Spider`, deprecated in + 2.13.0, is removed and no longer called. Use :meth:`~scrapy.Spider.start` + instead, or both to maintain support for lower Scrapy versions. + (:issue:`7490`) + +- Support for ``process_start_requests()`` methods of :ref:`spider middlewares + `, deprecated in 2.13.0, is removed. Use + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, + or both to maintain support for lower Scrapy versions. + (:issue:`7490`) + +- Support for synchronous ``process_spider_output()`` methods of spider + middlewares, deprecated in Scrapy 2.13.0, is removed. You should upgrade + the affected middlewares to have asynchronous ``process_spider_output()`` + methods. + (:issue:`7504`) + +- The ``spider`` arguments of the following methods of + :class:`~scrapy.core.scraper.Scraper`, deprecated in Scrapy 2.13.0, are + removed: + + - ``close_spider()`` + + - ``enqueue_scrape()`` + + - ``handle_spider_error()`` + + - ``handle_spider_output()`` + + (:issue:`7487`) + +- HTTP/1.0 support code, deprecated in Scrapy 2.13.0, is removed. This + includes: + + - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler`` + + - The ``scrapy.core.downloader.webclient`` module. + + - The ``DOWNLOADER_HTTPCLIENTFACTORY`` setting. + + (:issue:`7486`) + +- The following functions, deprecated in Scrapy 2.13.0, are removed, you + should import them from :mod:`w3lib.url` directly instead: + + - ``scrapy.utils.url.add_or_replace_parameter()`` + + - ``scrapy.utils.url.add_or_replace_parameters()`` + + - ``scrapy.utils.url.any_to_uri()`` + + - ``scrapy.utils.url.canonicalize_url()`` + + - ``scrapy.utils.url.file_uri_to_path()`` + + - ``scrapy.utils.url.is_url()`` + + - ``scrapy.utils.url.parse_data_uri()`` + + - ``scrapy.utils.url.parse_url()`` + + - ``scrapy.utils.url.path_to_file_uri()`` + + - ``scrapy.utils.url.safe_download_url()`` + + - ``scrapy.utils.url.safe_url_string()`` + + - ``scrapy.utils.url.url_query_cleaner()`` + + - ``scrapy.utils.url.url_query_parameter()`` + + (:issue:`7487`) + +- The following test-related code, deprecated in Scrapy 2.13.0, is removed: + + - the ``scrapy.utils.testproc`` module + + - the ``scrapy.utils.testsite`` module + + - ``scrapy.utils.test.assert_gcs_environ()`` + + - ``scrapy.utils.test.get_ftp_content_and_delete()`` + + - ``scrapy.utils.test.get_gcs_content_and_delete()`` + + - ``scrapy.utils.test.mock_google_cloud_storage()`` + + - ``scrapy.utils.test.skip_if_no_boto()`` + + - ``scrapy.utils.test.TestSpider`` + + (:issue:`7487`) + +- ``scrapy.utils.versions.scrapy_components_versions()``, deprecated in + Scrapy 2.13.0, is removed, you can use + :func:`scrapy.utils.versions.get_versions` instead. + (:issue:`7487`) + +- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` and + ``scrapy.utils.url.escape_ajax()``, deprecated in Scrapy 2.13.0, are + removed. + (:issue:`7487`) + +- The ``__init__()`` method of priority queue classes (see + :setting:`SCHEDULER_PRIORITY_QUEUE`) now needs to support a keyword-only + ``start_queue_cls`` parameter, not supporting it was deprecated in Scrapy + 2.13.0. + (:issue:`7487`) + +- ``scrapy.spiders.init.InitSpider``, deprecated in Scrapy 2.13.0, is + removed. + (:issue:`7487`) + +New features +~~~~~~~~~~~~ + +- New features and improvements for + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`: + + - Support for proxies. + + - Support for the :reqmeta:`download_latency` meta key. + + - Support for :attr:`Response.certificate + `. + + - Default headers set by the ``httpx`` library are no longer added to + requests. + + (:issue:`7441`, :issue:`7524`) + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now + skips HTTPS proxy certificate verification when the + :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting is set to ``False``. + (:issue:`7496`) + +Improvements +~~~~~~~~~~~~ + +- :func:`time.monotonic` is used instead of :func:`time.time` to calculate + elapsed time in various places. + (:issue:`7377`) + +- Improved extraction of the file extension from the URL in + :class:`~scrapy.pipelines.files.FilesPipeline`. + (:issue:`4225`, :issue:`7414`) + +- Other code refactoring and improvements. + (:issue:`7401`) + +Bug fixes +~~~~~~~~~ + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now + raises an exception when a request has an ``https://`` destination and an + ``https://`` proxy, which is not supported by this handler. Previously it + tried to connect to the proxy via HTTP in this case. + (:issue:`7496`) + +- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now + raises an exception for requests with ``http://`` URLs instead of trying to + connect, which is not supported by this handler. + (:issue:`7496`) + +- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` no longer + adds the ``:status`` pseudo-header to :attr:`Response.headers + `. + (:issue:`7441`) + +- Fixed :func:`scrapy.utils.response.open_in_browser` removing the ```` + tag when adding the ```` tag. + (:issue:`7459`) + +Documentation +~~~~~~~~~~~~~ + +- Documented that + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + doesn't support HTTPS proxies for HTTPS destinations and that + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` doesn't + support proxies at all. + (:issue:`7496`) + +- Added an example of using + :class:`logging.handlers.TimedRotatingFileHandler` to rotate Scrapy logs. + (:issue:`3628`, :issue:`7501`) + +- Added a ``CITATION.cff`` file. + (:issue:`7502`, :issue:`7519`) + +- Mentioned ``DOWNLOADER_CLIENT_TLS_METHOD`` in :ref:`bans`. + (:issue:`5232`, :issue:`7518`) + +- Other documentation improvements and fixes. + (:issue:`7417`, + :issue:`7463`, + :issue:`7472`, + :issue:`7480`, + :issue:`7489`, + :issue:`7503`, + :issue:`7507`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added tests that connect to https://books.toscrape.com/ to test the + behavior with a real website. These tests are marked with the + ``requires_internet`` pytest mark and can be skipped with e.g. + ``-m 'not requires_internet'`` if you cannot or don't want to run them. + (:issue:`7520`) + +- Type hints improvements and fixes. + (:issue:`7492`, :issue:`7532`) + +- CI and test improvements and fixes. + (:issue:`7441`, :issue:`7466`, :issue:`7491`, :issue:`7496`) + +.. _release-2.15.2: + +Scrapy 2.15.2 (2026-04-28) +-------------------------- + +Bug fixes +~~~~~~~~~ + +- Fixed links in https://docs.scrapy.org/llms.txt (:issue:`7467`) + +.. _release-2.15.1: + +Scrapy 2.15.1 (2026-04-23) +-------------------------- + +Bug fixes +~~~~~~~~~ + +- Sharing of the SSL context between multiple connections, introduced in + Scrapy 2.15.0, is reverted as it caused problems and wasn't actually + needed. + (:issue:`7445`, :issue:`7450`) + +- Fixed :meth:`scrapy.settings.BaseSettings.getwithbase` failing on keys with + dots that aren't import names. It now works the way it worked before Scrapy + 2.15.0, without trying to match class objects and import path. A separate + method, + :func:`~scrapy.settings.BaseSettings.get_component_priority_dict_with_base`, + was added that does that, and it is now used for :ref:`component priority + dictionaries `. + (:issue:`7426`, :issue:`7449`) + +- Documentation rendering improvements. + (:issue:`7452`, :issue:`7454`) + +.. _release-2.15.0: + +Scrapy 2.15.0 (2026-04-09) +-------------------------- + +Highlights: + +- Experimental support for running without a Twisted reactor + +- Experimental ``httpx``-based download handler + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The built-in HTTP :ref:`download handlers ` now + raise Scrapy-specific exceptions instead of implementation-specific ones, + see :ref:`download-handlers-exceptions`. This can affect user code that + handles downloader exceptions, such as ``process_exception()`` methods of + custom :ref:`downloader middlewares `. + (:issue:`7208`) + +- In order to fix a long-standing bug with handling of asynchronous storages, + the following changes were made to media pipeline classes, which can impact + some of the user code that subclasses them or calls their methods directly: + + - overrides of :meth:`scrapy.pipelines.media.MediaPipeline.media_downloaded` + and :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` can now + return coroutines + + - :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded`, + :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` and + :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded` now + return coroutines + + (:issue:`2183`, :issue:`6369`, :issue:`7182`) + +- ``Request`` and ``Response`` objects: ``__slots__`` and setter changes: + + - :class:`scrapy.http.Request` and :class:`scrapy.http.Response` now + define ``__slots__``. Assigning arbitrary attributes to instances (for + example, ``response.foo = 1``) will raise ``AttributeError``. Store + per-request/response data in the request/response ``meta`` mapping + instead of attaching new attributes to the objects. + + - If you maintain custom ``Request`` or ``Response`` subclasses that + relied on dynamic instance attributes, either add ``'__dict__'`` to + your subclass ``__slots__`` to allow dynamic attributes, or migrate + per-instance state to ``meta`` or explicit documented attributes. + + - The setters for ``headers``, ``flags`` and ``cookies`` no longer coerce + falsy values into ``None``. For example, ``request.headers = {}`` now + stores an empty :class:`scrapy.http.headers.Headers` instance (not + ``None``), and ``request.flags = []`` remains an empty list instead of + being set to ``None``. Update code that relied on ``is None`` checks or + the previous coercion behaviour. + + (:issue:`7036`, :issue:`7367`, :issue:`7374`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The context factory class set as the value of the + ``DOWNLOADER_CLIENTCONTEXTFACTORY`` setting is now required to support the + ``method`` argument of ``__init__()``, recommended since Scrapy 1.2.0. + (:issue:`7353`) + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.mail.MailSender`` is deprecated. Please use :mod:`smtplib`, + :mod:`twisted.mail.smtp` or other 3rd party email libraries. + (:issue:`7249`, :issue:`7263`) + +- The ``scrapy.extensions.statsmailer.StatsMailer`` extension is deprecated. + You can instead implement your own notifications by handling the + :signal:`spider_closed` signal. + (:issue:`7249`, :issue:`7263`) + +- The ``MEMUSAGE_NOTIFY_MAIL`` setting is deprecated. You can instead + implement your own notifications by handling the + :signal:`memusage_warning_reached` and :signal:`spider_closed` signals. + (:issue:`7249`, :issue:`7263`) + +- The ``DNS_RESOLVER`` setting was renamed to :setting:`TWISTED_DNS_RESOLVER` + and the old name is deprecated. + (:issue:`7350`, :issue:`7361`) + +- The ``DOWNLOADER_CLIENTCONTEXTFACTORY`` setting is deprecated. If you were + using it to switch to + ``scrapy.core.downloader.contextfactory.BrowserLikeContextFactory``, please + use the new :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting instead. If you + cannot use the default context factory for some other reason, please + subclass the :ref:`download handler ` instead. + (:issue:`7352`, :issue:`7379`) + +- ``scrapy.core.downloader.contextfactory.BrowserLikeContextFactory`` is + deprecated. You can set the new :setting:`DOWNLOAD_VERIFY_CERTIFICATES` + setting to ``True`` instead. + (:issue:`7379`) + +- The following implementation details of the context factory handling code + are deprecated: + + - ``scrapy.core.downloader.contextfactory.AcceptableProtocolsContextFactory`` + + - ``scrapy.core.downloader.contextfactory.load_context_factory_from_settings()`` + + - ``scrapy.core.downloader.contextfactory.ScrapyClientContextFactory`` + + - ``scrapy.core.downloader.tls.ScrapyClientTLSOptions`` + + (:issue:`7353`, :issue:`7391`) + +- Passing :class:`str` instead of :class:`bytes` to + :class:`scrapy.utils.sitemap.Sitemap` and + :func:`scrapy.utils.sitemap.sitemap_urls_from_robots` is deprecated. + (:issue:`7007`) + +- ``scrapy.utils.misc.walk_modules()`` is deprecated. You can use + :func:`scrapy.utils.misc.walk_modules_iter` instead. + (:issue:`7388`) + +- ``scrapy.shell.Shell.inthread`` is deprecated. You can use + :attr:`scrapy.shell.Shell.fetch_available` instead to check if + :func:`~scrapy.shell.Shell.fetch` can be used. + (:issue:`7395`) + +- ``scrapy.commands.ScrapyCommand.set_crawler()`` is deprecated. + (:issue:`7276`) + +New features +~~~~~~~~~~~~ + +- Added an *experimental* mode for running Scrapy without installing a + Twisted reactor: set :setting:`TWISTED_REACTOR_ENABLED` to ``False`` to + enable it. This mode has limitations, refer to :ref:`its documentation + ` for details. As long as it's experimental, its + behavior and related features and APIs may change in future Scrapy releases + in a breaking way. + (:issue:`6219`, + :issue:`7185`, + :issue:`7186`, + :issue:`7187`, + :issue:`7188`, + :issue:`7190`, + :issue:`7197`, + :issue:`7199`, + :issue:`7209`, + :issue:`7228`, + :issue:`7355`, + :issue:`7366`, + :issue:`7385`, + :issue:`7395`) + +- Added the :func:`scrapy.utils.reactorless.is_reactorless` function that + checks if there is a running asyncio event loop but no Twisted reactor. + (:issue:`7185`, :issue:`7199`) + +- Changed :func:`scrapy.utils.asyncio.is_asyncio_available` to return + ``True`` if there is a running asyncio loop, even if no Twisted reactor is + installed. + (:issue:`7185`, :issue:`7199`) + +- Added an *experimental* download handler that uses the httpx_ library and + doesn't require a Twisted reactor: + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. As + long as it's experimental, its behavior may change in future Scrapy + releases in a breaking way. + (:issue:`6805`, :issue:`7239`, :issue:`7368`, :issue:`7384`) + + .. _httpx: https://www.python-httpx.org/ + +- Added the :setting:`DOWNLOAD_BIND_ADDRESS` setting as a global counterpart + to the per-request :reqmeta:`bindaddress` meta key. + (:issue:`7266`, :issue:`7283`) + +- Added the :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting that can be set + to ``True`` to make Scrapy abort HTTPS requests when the server certificate + is invalid or doesn't match the domain. + (:issue:`7379`) + +- The built-in HTTP :ref:`download handlers ` now + raise Scrapy-specific exceptions instead of implementation-specific ones, + to allow unified handling of similar problems caused by different + implementations. The default value of the :setting:`RETRY_EXCEPTIONS` + setting was updated replacing Twisted-specific exceptions with these new + ones. The exceptions: + + - :exc:`~scrapy.exceptions.CannotResolveHostError` + + - :exc:`~scrapy.exceptions.DownloadCancelledError` + + - :exc:`~scrapy.exceptions.DownloadConnectionRefusedError` + + - :exc:`~scrapy.exceptions.DownloadFailedError` + + - :exc:`~scrapy.exceptions.DownloadTimeoutError` + + - :exc:`~scrapy.exceptions.ResponseDataLossError` + + - :exc:`~scrapy.exceptions.UnsupportedURLSchemeError` + + (:issue:`7208`) + +- Added the :signal:`memusage_warning_reached` signal emitted by the + :class:`~scrapy.extensions.memusage.MemoryUsage` extension when the memory + usage reaches :setting:`MEMUSAGE_WARNING_MB`. + (:issue:`7249`, :issue:`7263`) + +- Added + :meth:`Headers.to_tuple_list() ` + that returns headers as a list of ``(key, value)`` tuples. + (:issue:`7239`) + +- :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler` now uses the + download handler configured for the ``"https"`` scheme to make requests + instead of always using + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + (:issue:`7369`, :issue:`7370`) + +- Added :func:`scrapy.utils.misc.walk_modules_iter` as a replacement for + ``scrapy.utils.misc.walk_modules()`` that returns an iterable instead of a + list. + (:issue:`7388`) + +Improvements +~~~~~~~~~~~~ + +- :func:`asyncio.to_thread` is now used instead of + :func:`twisted.internet.threads.deferToThread` in the built-in feed + storages, media pipeline storages and the + :func:`scrapy.utils.decorators.inthread` decorator when available. + (:issue:`7183`, :issue:`7184`, :issue:`7349`) + +- Improved memory footprint of :class:`~scrapy.Request` and + :class:`~scrapy.http.Response` objects by adding ``__slots__`` and omitting + empty lists and dicts in some internal attributes. + (:issue:`7036`, :issue:`7367`, :issue:`7374`) + +- :class:`~scrapy.core.downloader.contextfactory._ScrapyClientContextFactory` + no longer mutates the SSL context, to avoid the behavior that was + deprecated in pyOpenSSL 25.1.0. + (:issue:`6859`, :issue:`7353`) + +- Improved memory usage of :class:`~scrapy.spiders.sitemap.SitemapSpider` and + :class:`scrapy.utils.sitemap.Sitemap`. + (:issue:`3529`, :issue:`7007`) + +- Improved the scheduling behavior of + :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` when crawling + multiple domains. + (:issue:`7293`, :issue:`7351`) + +- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now handle + TLS verbose logging (see :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING`) + directly instead of relying on + :class:`~scrapy.core.downloader.contextfactory._ScrapyClientContextFactory`. + (:issue:`7387`) + +- The server certificate verification code now correctly handles certificates + with IP addresses in ``subjectAltName``. + (:issue:`7353`) + +- Improved reliability of :func:`scrapy.utils.trackref.get_oldest`. + (:issue:`1758`, :issue:`7375`) + +- Other code refactoring and improvements. + (:issue:`7210`, :issue:`7238`, :issue:`7376`, :issue:`7386`, :issue:`7395`, + :issue:`7405`, :issue:`7410`) + +Bug fixes +~~~~~~~~~ + +- :ref:`Media pipelines ` should now wait for uploads + to asynchronous storages (e.g. + :class:`~scrapy.pipelines.files.S3FilesStore`) to complete. + (:issue:`2183`, :issue:`6369`, :issue:`7182`) + +- Fixed merging ``*_BASE`` settings (e.g. merging + :setting:`DOWNLOADER_MIDDLEWARES` with + :setting:`DOWNLOADER_MIDDLEWARES_BASE`) when a component is referred to by + a class object in one setting and by a string import path in the other one. + (:issue:`6912`, :issue:`6993`) + +- ``scrapy runspider`` and ``scrapy crawl`` now set the exit code to 1 if an + exception happened early (this was broken since Scrapy 2.13.0). + (:issue:`6820`, :issue:`7255`) + +- Fixed repeated warnings about data loss (see + :setting:`DOWNLOAD_FAIL_ON_DATALOSS`) not being suppressed in + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + (:issue:`7222`) + +- Improved FTP connection management in + :class:`scrapy.pipelines.files.FTPFilesStore`. + (:issue:`7256`) + +- Fixed the ``spider`` variable in the :ref:`shell `, which + wasn't available since Scrapy 2.13.0. + (:issue:`7395`) + +Documentation +~~~~~~~~~~~~~ + +- The ``llms.txt`` and ``llms-full.txt`` files and Markdown versions of pages + are now generated when the HTML documentation is built. + (:issue:`7380`) + +- Added a "Copy as Markdown" button to the HTML documentation. + (:issue:`7380`) + +- Added :ref:`docs for using Pydantic models as items `. + (:issue:`6955`, :issue:`6966`) + +- Documented :ref:`job directory contents `. + (:issue:`4842`, :issue:`5260`) + +- Improved docs for :attr:`~scrapy.Request.dont_filter`. + (:issue:`6398`, :issue:`7245`) + +- Clarified that settings related to :setting:`TWISTED_DNS_RESOLVER` are only + taken into account if the selected resolver supports them. + (:issue:`7385`) + +- Other documentation improvements and fixes. + (:issue:`7248`, :issue:`7274`, :issue:`7406`, :issue:`7408`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Added the ``no-reactor`` test environment that doesn't install a Twisted + reactor and uses ``pytest-asyncio`` instead of ``pytest-twisted`` to run + asynchronous test functions. + (:issue:`6952`, :issue:`7189`, :issue:`7233`, :issue:`7234`, :issue:`7254`, + :issue:`7259`) + +- Fixed running tests with ``pytest-xdist``. + (:issue:`7216`, :issue:`7257`) + +- Type hints improvements and fixes. + (:issue:`7300`, :issue:`7331`) + +- CI and test improvements and fixes. + (:issue:`7060`, + :issue:`7223`, + :issue:`7232`, + :issue:`7241`, + :issue:`7250`, + :issue:`7256`, + :issue:`7276`, + :issue:`7277`, + :issue:`7279`, + :issue:`7329`, + :issue:`7363`, + :issue:`7381`, + :issue:`7402`) + +.. _release-2.14.2: + +Scrapy 2.14.2 (2026-03-12) +-------------------------- + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Values from the ``Referrer-Policy`` header of HTTP responses are no longer + executed as Python callables. See the `cwxj-rr6w-m6w7`_ security advisory + for details. + + .. _cwxj-rr6w-m6w7: https://github.com/scrapy/scrapy/security/advisories/GHSA-cwxj-rr6w-m6w7 + +- In line with the `standard + `__, 301 redirects of + ``POST`` requests are converted into ``GET`` requests. + + Converting to a ``GET`` request implies not only a method change, but also + omitting the body and ``Content-*`` headers in the redirect request. On + cross-origin redirects (for example, cross-domain redirects), this is + effectively a security bug fix for scenarios where the body contains + secrets. + +Deprecations +~~~~~~~~~~~~ + +- Passing a response URL string as the first positional argument to + :meth:`scrapy.spidermiddlewares.referer.RefererMiddleware.policy` is + deprecated. Pass a :class:`~scrapy.http.Response` instead. + + The parameter has also been renamed to ``response`` to reflect this change. + The old parameter name (``resp_or_url``) is deprecated. + +New features +~~~~~~~~~~~~ + +- Added a new setting, :setting:`REFERRER_POLICIES`, to allow customizing + supported referrer policies. + +Bug fixes +~~~~~~~~~ + +- Made additional redirect scenarios convert to ``GET`` in line with the + `standard `__: + + - Only ``POST`` 302 redirects are converted into ``GET`` requests; other + methods are preserved. + + - ``HEAD`` 303 redirects are not converted into ``GET`` requests. + + - ``GET`` 303 redirects do not have their body or standard ``Content-*`` + headers removed. + +- Redirects where the original request body is dropped now also have their + ``Content-Encoding``, ``Content-Language`` and ``Content-Location`` headers + removed, in addition to the ``Content-Type`` and ``Content-Length`` headers + that were already being removed. + +- Redirects now preserve the source URL fragment if the redirect URL does not + include one. This is useful when using browser-based download handlers, + such as `scrapy-playwright`_ or `scrapy-zyte-api`_, while letting Scrapy + handle redirects. + + .. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright + .. _scrapy-zyte-api: https://scrapy-zyte-api.readthedocs.io/en/latest/ + +- The ``Referer`` header is now removed on redirect if + :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` is disabled. + +- The handling of the ``Referer`` header on redirects now takes into account + the ``Referer-Policy`` header of the response that triggers the redirect. + +.. _release-2.14.1: + +Scrapy 2.14.1 (2026-01-12) +-------------------------- + +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.utils.defer.maybeDeferred_coro()`` is deprecated. (:issue:`7212`) + +Bug fixes +~~~~~~~~~ + +- Fixed custom stats collectors that require a ``spider`` argument in their + ``open_spider()`` and ``close_spider()`` methods not receiving the + argument when called by the engine. + + Note, however, that the ``spider`` argument is now deprecated and will stop + being passed in a future version of Scrapy. + + (:issue:`7213`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Replaced deprecated ``codecov/test-results-action@v1`` GitHub Action with + ``codecov/codecov-action@v5``. + (:issue:`7180`, :issue:`7215`) + +.. _release-2.14.0: + +Scrapy 2.14.0 (2026-01-05) +-------------------------- + +Highlights: + +- More coroutine-based replacements for Deferred-based APIs + +- The default priority queue is now ``DownloaderAwarePriorityQueue`` + +- Dropped support for Python 3.9 and PyPy 3.10 + +- Improved and documented the API for custom download handlers + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for Python 3.9. + (:issue:`7121`) + +- Dropped support for PyPy 3.10. + (:issue:`7050`) + +- Increased the minimum versions of the following dependencies: + + - lxml_: 4.6.0 → 4.6.4 + + - Pillow_ (optional dependency): 8.0.0 → 8.3.2 + + - botocore_ (optional dependency): 1.4.87 → 1.13.45 + +- Restored support for ``brotlicffi`` dropped in Scrapy 2.13.4. Its minimum + supported version is now ``1.2.0.0``. + (:issue:`7160`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- If you set the :setting:`TWISTED_REACTOR` setting to a :ref:`non-asyncio + value ` at the :ref:`spider level `, you + may now need to set the :setting:`FORCE_CRAWLER_PROCESS` setting to + ``True`` when running Scrapy via :ref:`its command-line tool + ` to avoid a reactor mismatch exception. + (:issue:`6845`) + +- The ``log_count/*`` stats no longer count some of the early messages that + they counted before. While the earliest log messages, emitted before the + counter is initialized, were never counted, the counter initialization now + happens later than in previous Scrapy versions. You may need to adjust + expected values if you retrieve and compare values of these stats in your + code. + (:issue:`7046`) + +- The classes listed below are now :term:`abstract base classes `. They cannot be instantiated directly and their subclasses + need to override the abstract methods listed below to be able to be + instantiated. If you previously instantiated these classes directly, you + will now need to subclass them and provide trivial (e.g. empty) + implementations for the abstract methods. + + - :class:`scrapy.commands.ScrapyCommand` + + - :meth:`~scrapy.commands.ScrapyCommand.run` + + - :meth:`~scrapy.commands.ScrapyCommand.short_desc` + + - :class:`scrapy.exporters.BaseItemExporter` + + - :meth:`~scrapy.exporters.BaseItemExporter.export_item` + + - :class:`scrapy.extensions.feedexport.BlockingFeedStorage` + + - :meth:`~scrapy.extensions.feedexport.BlockingFeedStorage._store_in_thread` + + - :class:`scrapy.middleware.MiddlewareManager` + + - :meth:`~scrapy.middleware.MiddlewareManager._get_mwlist_from_settings` + + - :class:`scrapy.spidermiddlewares.referer.ReferrerPolicy` + + - :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer` + + (:issue:`6930`) + +- Scrapy no longer passes a ``spider`` argument to any methods of the + :setting:`stats collector `. It wasn't passed in many of the + calls even in older Scrapy versions, so we don't expect existing custom + stats collector implementations to require a ``spider`` argument. If your + implementation needs a :class:`~scrapy.Spider` instance, you can get it + from the :class:`~scrapy.crawler.Crawler` instance passed to the + constructor. + (:issue:`7011`) + +- :class:`scrapy.middleware.MiddlewareManager` no longer includes code for + handling ``open_spider()`` and ``close_spider()`` component methods. As + this code was only used for pipelines it was moved into + :class:`scrapy.pipelines.ItemPipelineManager`. This change should only + affect custom subclasses of :class:`~scrapy.middleware.MiddlewareManager`. + The following code was moved: + + - ``scrapy.middleware.MiddlewareManager.open_spider()`` + + - ``scrapy.middleware.MiddlewareManager.close_spider()`` + + - Code in ``scrapy.middleware.MiddlewareManager._add_middleware()`` that + processes ``open_spider()`` and ``close_spider()`` component methods. + + (:issue:`7006`) + +- :meth:`scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware.process_request` + now returns a coroutine, previously it returned a + :class:`~twisted.internet.defer.Deferred` object or ``None``. The + ``robot_parser()`` method was also changed to return a coroutine. This + change only impacts code that subclasses + :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` or + calls its methods directly. + (:issue:`6802`) + +- The built-in :ref:`download handlers ` have been + refactored, changing the signatures of their methods. This change should + only affect user code that subclasses any of these handlers or calls their + methods directly. + (:issue:`6778`, :issue:`7164`) + +- :meth:`scrapy.pipelines.media.MediaPipeline.process_item` now returns a + coroutine, previously it returned a + :class:`~twisted.internet.defer.Deferred` object. This + change only impacts code that calls this method directly. + (:issue:`7177`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- The ``from_settings()`` method of the following components, deprecated in + Scrapy 2.12.0, is removed. You should use ``from_crawler()`` instead. + + - :class:`scrapy.dupefilters.RFPDupeFilter` + - :class:`scrapy.mail.MailSender` + - :class:`scrapy.middleware.MiddlewareManager` + - :class:`scrapy.core.downloader.contextfactory.ScrapyClientContextFactory` + - :class:`scrapy.pipelines.files.FilesPipeline` + - :class:`scrapy.pipelines.images.ImagesPipeline` + + (:issue:`7126`) + +- Scrapy no longer calls ``from_settings()`` methods of 3rd-party + :ref:`components `, deprecated in Scrapy 2.12.0. You + should define a ``from_crawler()`` method instead. + (:issue:`7126`) + +- The initialization flow of :class:`scrapy.pipelines.media.MediaPipeline` + and its subclasses was simplified, it now mandates ``from_crawler()`` + methods and ``crawler`` arguments of ``__init__()`` methods. Not using + these was deprecated in Scrapy 2.12.0. + (:issue:`7126`) + +- The ``REQUEST_FINGERPRINTER_IMPLEMENTATION`` setting, deprecated in Scrapy + 2.12.0, is removed. + (:issue:`7126`) + +- The ``scrapy.utils.misc.create_instance()`` function, deprecated in Scrapy + 2.12.0, is removed. Use :func:`scrapy.utils.misc.build_from_crawler` + instead. + (:issue:`7126`) + +- The ``scrapy.core.downloader.Downloader._get_slot_key()`` function, + deprecated in Scrapy 2.12.0, is removed. Use + :meth:`scrapy.core.downloader.Downloader.get_slot_key` instead. + (:issue:`7126`) + +- The ``scrapy.twisted_version`` attribute, deprecated in Scrapy 2.12.0, is + removed. You should instead use the :attr:`twisted.version` attribute + directly. + (:issue:`7126`) + +- The following utility functions, deprecated in Scrapy 2.12.0, are removed: + + - ``scrapy.utils.defer.process_chain_both()`` + - ``scrapy.utils.python.equal_attributes()`` + - ``scrapy.utils.python.flatten()`` + - ``scrapy.utils.python.iflatten()`` + - ``scrapy.utils.request.request_authenticate()`` + - ``scrapy.utils.test.assert_samelines()`` + + (:issue:`7126`) + +- ``scrapy.utils.serialize.ScrapyJSONDecoder``, deprecated in Scrapy 2.12.0, + is removed. + (:issue:`7126`) + +- The ``scrapy.extensions.feedexport.build_storage()`` function, deprecated + in Scrapy 2.12.0, is removed, you can instead call the builder callable + directly. + (:issue:`7126`) + +- ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware``, deprecated in + Scrapy 2.11.2, is removed. + :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` should be + used instead. + (:issue:`6926`) + +Deprecations +~~~~~~~~~~~~ + +- The following methods that return a + :class:`~twisted.internet.defer.Deferred` are deprecated in favor of their + coroutine-based replacements: + + - :class:`scrapy.core.downloader.handlers.DownloadHandlers` + + - ``download_request()`` (use + :meth:`~scrapy.core.downloader.handlers.DownloadHandlers.download_request_async`) + + - :class:`scrapy.core.downloader.middleware.DownloaderMiddlewareManager` + + - ``download()`` (use + :meth:`~scrapy.core.downloader.middleware.DownloaderMiddlewareManager.download_async`) + + - :class:`scrapy.core.engine.ExecutionEngine` + + - ``start()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.start_async`) + + - ``stop()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.stop_async`) + + - ``close()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.close_async`) + + - ``open_spider()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.open_spider_async`) + + - ``close_spider()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.close_spider_async`) + + - ``download()`` (use + :meth:`~scrapy.core.engine.ExecutionEngine.download_async`) + + - :class:`scrapy.core.scraper.Scraper` + + - ``open_spider()`` (use + :meth:`~scrapy.core.scraper.Scraper.open_spider_async`) + + - ``call_spider()`` (use + :meth:`~scrapy.core.scraper.Scraper.call_spider_async`) + + - ``close_spider()`` (use + :meth:`~scrapy.core.scraper.Scraper.close_spider_async`) + + - ``handle_spider_output()`` (use + :meth:`~scrapy.core.scraper.Scraper.handle_spider_output_async`) + + - ``start_itemproc()`` (use + :meth:`~scrapy.core.scraper.Scraper.start_itemproc_async`) + + - :class:`scrapy.core.spidermw.SpiderMiddlewareManager` + + - ``scrape_response()`` (use + :meth:`~scrapy.core.spidermw.SpiderMiddlewareManager.scrape_response_async`) + + - :class:`scrapy.crawler.Crawler` + + - ``stop()`` (use :meth:`~scrapy.crawler.Crawler.stop_async`) + + - :class:`scrapy.pipelines.ItemPipelineManager` + + - ``process_item()`` (use + :meth:`~scrapy.pipelines.ItemPipelineManager.process_item_async`) + + - ``open_spider()`` (use + :meth:`~scrapy.pipelines.ItemPipelineManager.open_spider_async`) + + - ``close_spider()`` (use + :meth:`~scrapy.pipelines.ItemPipelineManager.close_spider_async`) + + - :class:`scrapy.signalmanager.SignalManager` + + - ``send_catch_log_deferred()`` (use + :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_async`) + + - ``scrapy.utils.signal.send_catch_log_deferred()`` (use + :func:`scrapy.utils.signal.send_catch_log_async`) + + (:issue:`6791`, :issue:`6842`, :issue:`6979`, :issue:`6997`, :issue:`6999`, + :issue:`7005`, :issue:`7043`, :issue:`7069`, :issue:`7161`, :issue:`7164`) + +- The following spider attributes are deprecated in favor of settings: + + - ``download_maxsize`` (use :setting:`DOWNLOAD_MAXSIZE`) + + - ``download_timeout`` (use :setting:`DOWNLOAD_TIMEOUT`) + + - ``download_warnsize`` (use :setting:`DOWNLOAD_WARNSIZE`) + + - ``max_concurrent_requests`` (use :setting:`CONCURRENT_REQUESTS`) + + - ``user_agent`` (use :setting:`USER_AGENT`) + + (:issue:`6988`, :issue:`6994`, :issue:`7038`, :issue:`7039`, :issue:`7117`, + :issue:`7176`) + +- Returning a :class:`~twisted.internet.defer.Deferred` from the following + user-defined functions is deprecated in favor of defining them as coroutine + functions: + + - spider callbacks and errbacks (which was never officially supported and + may work incorrectly) + + - the ``process_request()``, ``process_response()`` and + ``process_exception()`` methods of custom downloader middlewares + + - the ``process_item()``, ``open_spider()`` and ``close_spider()`` methods + of custom pipelines + + - signal handlers + + - the ``download_request()`` and ``close()`` methods of custom download + handlers + + (:issue:`6718`, :issue:`6778`, :issue:`7069`, :issue:`7147`, :issue:`7148`, + :issue:`7149`, :issue:`7150`, :issue:`7151`, :issue:`7161`, :issue:`7164`, + :issue:`7179`) + +- Passing a ``spider`` argument to the following methods is deprecated: + + - :meth:`scrapy.core.spidermw.SpiderMiddlewareManager.process_start` + + - :meth:`scrapy.core.downloader.Downloader.fetch` + + - :meth:`scrapy.core.downloader.Downloader._get_slot` + + - :meth:`scrapy.core.downloader.handlers.DownloadHandlers.download_request` + + - all public methods of :class:`scrapy.statscollectors.StatsCollector` + + - :meth:`scrapy.spidermiddlewares.base.BaseSpiderMiddleware.process_spider_output` + + - :meth:`scrapy.spidermiddlewares.base.BaseSpiderMiddleware.process_spider_output_async` + + - all ``process_*()`` methods of built-in downloader middlewares + + - all ``process_*()`` methods of built-in spider middlewares + + - :meth:`scrapy.pipelines.media.MediaPipeline.open_spider` + + - :meth:`scrapy.pipelines.media.MediaPipeline.process_item` + + (:issue:`6750`, :issue:`6927`, :issue:`6984`, :issue:`7006`, :issue:`7011`, + :issue:`7033`, :issue:`7037`, :issue:`7045`, :issue:`7178`) + +- Instantiating subclasses of :class:`scrapy.middleware.MiddlewareManager` + without a :class:`~scrapy.crawler.Crawler` instance is deprecated. + (:issue:`6984`) + +- For the following user-defined functions and methods requiring a ``spider`` + argument is deprecated, if you need a :class:`~scrapy.Spider` instance + inside them you should get it from the :class:`~scrapy.crawler.Crawler` + instance (you may need to refactor your code to save that instance in e.g. + the ``from_crawler()`` method): + + - the ``process_request()``, ``process_response()`` and + ``process_exception()`` methods of custom downloader middlewares + + - the ``process_spider_input()``, ``process_spider_output()``, + ``process_spider_output_async()`` and ``process_spider_exception()`` + methods of custom spider middlewares + + - the ``process_item()`` method of custom pipelines + + - the ``fetch()`` method of a custom :setting:`DOWNLOADER` + + (:issue:`6927`, :issue:`6984`, :issue:`7006`, :issue:`7037`) + +- The following things in custom download handlers are deprecated: + + - not having a ``lazy`` attribute (you should define it as ``True`` if you + want to keep the current behavior) + + - returning a :class:`~twisted.internet.defer.Deferred` from the + ``download_request()`` method (you should refactor it to return a + coroutine; you also need to remove the ``spider`` argument when doing + this) + + - not having a ``close()`` method, having a synchronous one or one that + returns a :class:`~twisted.internet.defer.Deferred` (you should refactor + it to return a coroutine or add an empty one if you don't have it) + + (:issue:`6778`, :issue:`7164`) + +- Custom implementations of :setting:`ITEM_PROCESSOR` should now define + ``process_item_async()``, ``open_spider_async()`` and + ``close_spider_async()`` methods instead of, or in addition to, + ``process_item()``, ``open_spider()`` and ``close_spider()``. + (:issue:`7005`, :issue:`7043`) + +- The ``CONCURRENT_REQUESTS_PER_IP`` setting is deprecated, use + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` instead. + (:issue:`6917`, :issue:`6921`) + +- The ``scrapy.core.downloader.handlers.http`` module is deprecated. You + should import + :class:`scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + directly instead of importing the + ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler`` alias. + (:issue:`7079`) + +- The ``scrapy.utils.decorators.defers()`` decorator is deprecated, you can + use :func:`twisted.internet.defer.maybeDeferred` directly or reimplement + this decorator in your code. + (:issue:`7164`) + +- ``scrapy.spiders.CrawlSpider._parse_response()`` is deprecated, use + :meth:`scrapy.spiders.CrawlSpider.parse_with_rules` instead. + (:issue:`4463`, :issue:`6804`) + +- The functions that add a delay to a Deferred are deprecated, their + underlying Twisted functions can be used instead, either directly if a + delay isn't needed, or with some explicit way to add a delay if it's + needed: + + - ``scrapy.utils.defer.mustbe_deferred()`` (you can use + :func:`twisted.internet.defer.maybeDeferred`) + + - ``scrapy.utils.defer.defer_succeed()`` (you can use + :func:`twisted.internet.defer.succeed`) + + - ``scrapy.utils.defer.defer_fail()`` (you can use + :func:`twisted.internet.defer.fail`) + + - ``scrapy.utils.defer.defer_result()`` (you can use + :func:`twisted.internet.defer.succeed` and + :func:`twisted.internet.defer.fail`) + + (:issue:`6937`) + +New features +~~~~~~~~~~~~ + +- Added :class:`scrapy.crawler.AsyncCrawlerProcess` and + :class:`scrapy.crawler.AsyncCrawlerRunner` as counterparts to + :class:`~scrapy.crawler.CrawlerProcess` and + :class:`~scrapy.crawler.CrawlerRunner` that offer coroutine-based APIs. + (:issue:`6789`, :issue:`6790`, :issue:`6796`, :issue:`6817`, :issue:`6845`, + :issue:`7034`) + +- Added coroutine counterparts to some of the Deferred-based APIs: + + - :class:`scrapy.core.downloader.handlers.DownloadHandlers` + + - :meth:`~scrapy.core.downloader.handlers.DownloadHandlers.download_request_async` + (to ``download_request()``) + + - :class:`scrapy.core.downloader.middleware.DownloaderMiddlewareManager` + + - :meth:`~scrapy.core.downloader.middleware.DownloaderMiddlewareManager.download_async` + (to ``download()``) + + - :class:`scrapy.core.engine.ExecutionEngine` + + - :meth:`~scrapy.core.engine.ExecutionEngine.start_async` (to + ``start()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.stop_async` (to + ``stop()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.close_async` (to + ``close()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.open_spider_async` (to + ``open_spider()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.close_spider_async` (to + ``close_spider()``) + + - :meth:`~scrapy.core.engine.ExecutionEngine.download_async` (to + ``download()``) + + - :class:`scrapy.core.scraper.Scraper` + + - :meth:`~scrapy.core.scraper.Scraper.open_spider_async` (to + ``open_spider()``) + + - :meth:`~scrapy.core.scraper.Scraper.close_spider_async` (to + ``close_spider()``) + + - :meth:`~scrapy.core.scraper.Scraper.start_itemproc_async` (to + ``start_itemproc()``) + + - :class:`scrapy.crawler.Crawler` + + - :meth:`~scrapy.crawler.Crawler.crawl_async` (to ``crawl()``) + + - :meth:`~scrapy.crawler.Crawler.stop_async` (to ``stop()``) + + - :class:`scrapy.pipelines.ItemPipelineManager` + + - :meth:`~scrapy.pipelines.ItemPipelineManager.process_item_async` (to + ``process_item()``) + + - :meth:`~scrapy.pipelines.ItemPipelineManager.open_spider_async` (to + ``open_spider()``) + + - :meth:`~scrapy.pipelines.ItemPipelineManager.close_spider_async` (to + ``close_spider()``) + + - :class:`scrapy.signalmanager.SignalManager` + + - :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_async` (to + ``send_catch_log_deferred()``) + + (:issue:`6781`, :issue:`6791`, :issue:`6792`, :issue:`6795`, :issue:`6801`, + :issue:`6817`, :issue:`6842`, :issue:`6997`, :issue:`7005`, :issue:`7043`, + :issue:`7069`,:issue:`7164`, :issue:`7202`) + +- The default value of the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is now + ``'scrapy.pqueues.DownloaderAwarePriorityQueue'``. + (:issue:`6924`, :issue:`6940`) + +- Added :class:`scrapy.extensions.logcount.LogCount`, an enabled-by-default + extension that is responsible for the ``log_count/*`` stats. Previously, + this code was in :class:`scrapy.crawler.Crawler` and couldn't be disabled. + (:issue:`7046`) + +- Added :meth:`scrapy.spiders.CrawlSpider.parse_with_rules` as a public + replacement for ``_parse_response()``. + (:issue:`4463`, :issue:`6804`) + +- Added :func:`scrapy.utils.asyncio.is_asyncio_available` as an alternative + to :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` with a + future-proof name and semantics. + (:issue:`6827`) + +- The API for :ref:`download handlers `, previously + undocumented, has been modernized and documented. An optional base class, + :class:`scrapy.core.downloader.handlers.base.BaseDownloadHandler`, has been + added to simplify writing custom download handlers that conform to the + current API. + (:issue:`4944`, :issue:`6778`, :issue:`7164`) + +- Added :func:`scrapy.utils.defer.ensure_awaitable`, which can be helpful to + call user-defined functions that can return coroutines, Deferreds or + values directly. + (:issue:`7005`) + +- The ``requests.seen`` file, written by + :class:`~scrapy.dupefilters.RFPDupeFilter` when :ref:`job persistence + ` is enabled, now uses line buffering to reduce data loss in + spider crashes. + (:issue:`6019`, :issue:`7094`) + +- Images downloaded by :class:`~scrapy.pipelines.images.ImagesPipeline` are + now automatically transposed based on EXIF data. + (:issue:`6525`, :issue:`6975`) + +Improvements +~~~~~~~~~~~~ + +- Refactored internal functions to use coroutines instead of Deferreds. + (:issue:`6795`, :issue:`6852`, :issue:`6855`, :issue:`6858`, :issue:`7159`) + +- Commands that don't need a :class:`~scrapy.crawler.CrawlerProcess` instance + no longer create it. + (:issue:`6824`) + +- Improved :command:`shell` help formatting when using IPython 9+. + (:issue:`6915`, :issue:`6980`) + +Bug fixes +~~~~~~~~~ + +- Setting :setting:`FILES_STORE` or :setting:`IMAGES_STORE` to ``None`` now + correctly disables the respective pipeline. + (:issue:`6964`, :issue:`6969`) + +- :class:`~scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware` now + uses the URL set in the ```` tag as the base URL when redirecting to + a relative URL. + (:issue:`7042`, :issue:`7047`) + +- Passing ``None`` as a value of the :reqmeta:`download_slot` request meta + key is now handled in the same way as not setting this meta key at all. + (:issue:`7172`) + +- Fixed parsing of the first line of ``robots.txt`` files that have a BOM. + (:issue:`6195`, :issue:`7095`) + +Documentation +~~~~~~~~~~~~~ + +- Added :ref:`documentation ` about download + handlers, their API and built-in handlers. + (:issue:`4944`, :issue:`7164`) + +- Added a section about the `scrapy-spider-metadata`_ library to the + :ref:`spider argument docs `. + (:issue:`6676`, :issue:`6957`, :issue:`7116`) + + .. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/ + +- Improved :ref:`the docs ` about coroutine-based + and Deferred-based APIs. + (:issue:`6800`, :issue:`7146`) + +- Other documentation improvements and fixes. + (:issue:`7058`, :issue:`7076`, :issue:`7109`, :issue:`7195`, :issue:`7198`) + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Switched from ``twisted.trial`` to ``pytest-twisted`` and replaced + remaining ``unittest`` and ``twisted.trial`` features with ``pytest`` ones. + (:issue:`6658`, :issue:`6873`, :issue:`6884`, :issue:`6938`) + +- Enabled fancy ``pytest`` asserts. + (:issue:`6888`) + +- Added `Sphinx Lint`_ to the ``pre-commit`` configuration. + (:issue:`6920`) + + .. _Sphinx Lint: https://github.com/sphinx-contrib/sphinx-lint + +- CI and test improvements and fixes. + (:issue:`6649`, + :issue:`6769`, + :issue:`6821`, + :issue:`6835`, + :issue:`6836`, + :issue:`6846`, + :issue:`6883`, + :issue:`6885`, + :issue:`6889`, + :issue:`6905`, + :issue:`6928`, + :issue:`6933`, + :issue:`6941`, + :issue:`6942`, + :issue:`6945`, + :issue:`6947`, + :issue:`6960`, + :issue:`6968`, + :issue:`6972`, + :issue:`6974`, + :issue:`6996`, + :issue:`7003`, + :issue:`7012`, + :issue:`7013`, + :issue:`7050`, + :issue:`7059`, + :issue:`7070`, + :issue:`7073`, + :issue:`7118`, + :issue:`7127`, + :issue:`7141`, + :issue:`7143`, + :issue:`7145`, + :issue:`7173`) + +- Code cleanups. + (:issue:`6803`, + :issue:`6838`, + :issue:`6849`, + :issue:`6875`, + :issue:`6876`, + :issue:`6892`, + :issue:`6930`, + :issue:`6949`, + :issue:`6970`, + :issue:`6977`, + :issue:`6986`, + :issue:`7008`, + :issue:`7177`) + +.. _release-2.13.4: + +Scrapy 2.13.4 (2025-11-17) +-------------------------- + +Security bug fixes +~~~~~~~~~~~~~~~~~~ + +- Improved protection against decompression bombs in + :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware` + for responses compressed using the ``br`` and ``deflate`` methods: if a + single compressed chunk would be larger than the response size limit (see + :setting:`DOWNLOAD_MAXSIZE`) when decompressed, decompression is no longer + carried out. This is especially important for the ``br`` (Brotli) method + that can provide a very high compression ratio. Please, see the + `CVE-2025-6176`_ and `GHSA-2qfp-q593-8484`_ security advisories for more + information. + (:issue:`7134`) + + .. _CVE-2025-6176: https://nvd.nist.gov/vuln/detail/CVE-2025-6176 + .. _GHSA-2qfp-q593-8484: https://github.com/advisories/GHSA-2qfp-q593-8484 + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- The minimum supported version of the optional ``brotli`` package is now + ``1.2.0``. + (:issue:`7134`) + +- The ``brotlicffi`` and ``brotlipy`` packages can no longer be used to + decompress Brotli-compressed responses. Please install the ``brotli`` + package instead. + (:issue:`7134`) + +Other changes +~~~~~~~~~~~~~ + +- Restricted the maximum supported Twisted version to ``25.5.0``, as Scrapy + currently uses some private APIs changed in later Twisted versions. + (:issue:`7142`) + +- Stopped setting the ``COVERAGE_CORE`` environment variable in tests, it + didn't have an effect but caused the ``coverage`` module to produce a + warning or an error. + (:issue:`7137`) + +- Removed the documentation build dependency on the deprecated + ``sphinx-hoverxref`` module. + (:issue:`6786`, :issue:`6922`) + +.. _release-2.13.3: + +Scrapy 2.13.3 (2025-07-02) +-------------------------- + +- Changed the values for :setting:`DOWNLOAD_DELAY` (from ``0`` to ``1``) and + :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (from ``8`` to ``1``) in the + default project template. + (:issue:`6597`, :issue:`6918`, :issue:`6923`) + +- Improved :class:`scrapy.core.engine.ExecutionEngine` logic related to + initialization and exception handling, fixing several cases where the + spider would crash, hang or log an unhandled exception. + (:issue:`6783`, :issue:`6784`, :issue:`6900`, :issue:`6908`, :issue:`6910`, + :issue:`6911`) + +- Fixed a Windows issue with :ref:`feed exports ` using + :class:`scrapy.extensions.feedexport.FileFeedStorage` that caused the file + to be created on the wrong drive. + (:issue:`6894`, :issue:`6897`) + +- Allowed running tests with Twisted 25.5.0+ again. Pytest 8.4.1+ is now + required for running tests in non-pinned envs as support for the new + Twisted version was added in that version. + (:issue:`6893`) + +- Fixed running tests with lxml 6.0.0+. + (:issue:`6919`) + +- Added a deprecation notice for + ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware`` to :ref:`the Scrapy + 2.11.2 release notes `. + (:issue:`6926`) + +- Updated :ref:`contribution docs ` to refer to ruff_ + instead of black_. + (:issue:`6903`) + +- Added ``.venv/`` and ``.vscode/`` to ``.gitignore``. + (:issue:`6901`, :issue:`6907`) + + +.. _release-2.13.2: + +Scrapy 2.13.2 (2025-06-09) +-------------------------- + +- Fixed a bug introduced in Scrapy 2.13.0 that caused results of request + errbacks to be ignored when the errback was called because of a downloader + error. + (:issue:`6861`, :issue:`6863`) + +- Added a note about the behavior change of + :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to its docs and + to the "Backward-incompatible changes" section of :ref:`the Scrapy 2.13.0 + release notes `. + (:issue:`6866`) + +- Improved the message in the exception raised by + :func:`scrapy.utils.test.get_reactor_settings` when there is no reactor + installed. + (:issue:`6866`) + +- Updated the :class:`scrapy.crawler.CrawlerRunner` examples in + :ref:`topics-practices` to install the reactor explicitly, to fix + reactor-related errors with Scrapy 2.13.0 and later. + (:issue:`6865`) + +- Fixed ``scrapy fetch`` not working with scrapy-poet_. + (:issue:`6872`) + +- Fixed an exception produced by :class:`scrapy.core.engine.ExecutionEngine` + when it's closed before being fully initialized. + (:issue:`6857`, :issue:`6867`) + +- Improved the README, updated the Scrapy logo in it. + (:issue:`6831`, :issue:`6833`, :issue:`6839`) + +- Restricted the Twisted version used in tests to below 25.5.0, as some tests + fail with 25.5.0. + (:issue:`6878`, :issue:`6882`) + +- Updated type hints for Twisted 25.5.0 changes. + (:issue:`6882`) + +- Removed the old artwork. + (:issue:`6874`) + + +.. _release-2.13.1: + +Scrapy 2.13.1 (2025-05-28) +-------------------------- + +- Give callback requests precedence over start requests when priority values + are the same. + + This makes changes from 2.13.0 to start request handling more intuitive and + backward compatible. For scenarios where all requests have the same + priorities, in 2.13.0 all start requests were sent before the first + callback request. In 2.13.1, same as in 2.12 and lower, start requests are + only sent when there are not enough pending callback requests to reach + concurrency limits. + + (:issue:`6828`) + +- Added a deepwiki_ badge to the README. (:issue:`6793`) + + .. _deepwiki: https://deepwiki.com/scrapy/scrapy + +- Fixed a typo in the code example of :ref:`start-requests-lazy`. + (:issue:`6812`, :issue:`6815`) + +- Fixed a typo in the :ref:`coroutine-support` section of the documentation. + (:issue:`6822`) + +- Made this page more prominently listed in PyPI project links. + (:issue:`6826`) + + +.. _release-2.13.0: + +Scrapy 2.13.0 (2025-05-08) +-------------------------- + +Highlights: + +- The asyncio reactor is now enabled by default + +- Replaced ``start_requests()`` (sync) with :meth:`~scrapy.Spider.start` + (async) and changed how it is iterated + +- Added the :reqmeta:`allow_offsite` request meta key + +- Spider middlewares that don't support asynchronous spider output are + deprecated + +- Added a base class for :ref:`universal spider middlewares + ` + +Modified requirements +~~~~~~~~~~~~~~~~~~~~~ + +- Dropped support for PyPy 3.9. + (:issue:`6613`) + +- Added support for PyPy 3.11. + (:issue:`6697`) + +Backward-incompatible changes +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- The default value of the :setting:`TWISTED_REACTOR` setting was changed + from ``None`` to + ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``. This value + was used in newly generated projects since Scrapy 2.7.0 but now existing + projects that don't explicitly set this setting will also use the asyncio + reactor. You can :ref:`change this setting in your project + ` to use a different reactor. + (:issue:`6659`, :issue:`6713`) + +- The iteration of start requests and items no longer stops once there are + requests in the scheduler, and instead runs continuously until all start + requests have been scheduled. + + To reproduce the previous behavior, see :ref:`start-requests-lazy`. + (:issue:`6729`) + +- An unhandled exception from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.open_spider` method of a + :ref:`spider middleware ` no longer stops the + crawl. + (:issue:`6729`) + +- In ``scrapy.core.engine.ExecutionEngine``: + + - The second parameter of ``open_spider()``, ``start_requests``, has been + removed. The start requests are determined by the ``spider`` parameter + instead (see :meth:`~scrapy.Spider.start`). + + - The ``slot`` attribute has been renamed to ``_slot`` and should not be + used. + + (:issue:`6729`) + +- In ``scrapy.core.engine``, the ``Slot`` class has been renamed to ``_Slot`` + and should not be used. + (:issue:`6729`) + +- The ``slot`` :ref:`telnet variable ` has been removed. + (:issue:`6729`) + +- In ``scrapy.core.spidermw.SpiderMiddlewareManager``, + ``process_start_requests()`` has been replaced by ``process_start()``. + (:issue:`6729`) + +- The now-deprecated ``start_requests()`` method, when it returns an iterable + instead of being defined as a generator, is now executed *after* the + :ref:`scheduler ` instance has been created. + (:issue:`6729`) + +- When using :setting:`JOBDIR`, :ref:`start requests ` are + now serialized into their own, ``s``-suffixed priority folders. You can set + :setting:`SCHEDULER_START_DISK_QUEUE` to ``None`` or ``""`` to change that, + but the side effects may be undesirable. See + :setting:`SCHEDULER_START_DISK_QUEUE` for details. + (:issue:`6729`) + +- The URL length limit, set by the :setting:`URLLENGTH_LIMIT` setting, is now + also enforced for start requests. + (:issue:`6777`) + +- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without + an installed reactor now raises an exception instead of installing a + reactor. This shouldn't affect normal Scrapy use cases, but it may affect + 3rd-party test suites that use Scrapy internals such as + :class:`~scrapy.crawler.Crawler` and don't install a reactor explicitly. If + you are affected by this change, you most likely need to install the + reactor before running Scrapy code that expects it to be installed. + (:issue:`6732`, :issue:`6735`) + +- The ``from_settings()`` method of + :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`, + deprecated in Scrapy 2.12.0, is removed earlier than the usual deprecation + period (this was needed because after the introduction of the + :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware` base class and + switching built-in spider middlewares to it those middlewares need the + :class:`~scrapy.crawler.Crawler` instance at run time). Please use + ``from_crawler()`` instead. + (:issue:`6693`) + +- ``scrapy.utils.url.escape_ajax()`` is no longer called when a + :class:`~scrapy.Request` instance is created. It was only useful for + websites supporting the ``_escaped_fragment_`` feature which most modern + websites don't support. If you still need this you can modify the URLs + before passing them to :class:`~scrapy.Request`. + (:issue:`6523`, :issue:`6651`) + +Deprecation removals +~~~~~~~~~~~~~~~~~~~~ + +- Removed old deprecated name aliases for some signals: + + - ``stats_spider_opened`` (use ``spider_opened`` instead) + + - ``stats_spider_closing`` and ``stats_spider_closed`` (use + ``spider_closed`` instead) + + - ``item_passed`` (use ``item_scraped`` instead) + + - ``request_received`` (use ``request_scheduled`` instead) + + (:issue:`6654`, :issue:`6655`) + +Deprecations +~~~~~~~~~~~~ + +- The ``start_requests()`` method of :class:`~scrapy.Spider` is deprecated, + use :meth:`~scrapy.Spider.start` instead, or both to maintain support for + lower Scrapy versions. + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- The ``process_start_requests()`` method of :ref:`spider middlewares + ` is deprecated, use + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, + or both to maintain support for lower Scrapy versions. + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- The ``__init__`` method of priority queue classes (see + :setting:`SCHEDULER_PRIORITY_QUEUE`) should now support a keyword-only + ``start_queue_cls`` parameter. + (:issue:`6752`) + +- Spider middlewares that don't support asynchronous spider output are + deprecated. The async iterable downgrading feature, needed for using such + middlewares with asynchronous callbacks and with other spider middlewares + that produce asynchronous iterables, is also deprecated. Please update all + such middlewares to support asynchronous spider output. (:issue:`6664`) + +- Functions that were imported from :mod:`w3lib.url` and re-exported in + :mod:`scrapy.utils.url` are now deprecated, you should import them from + :mod:`w3lib.url` directly. They are: + + - ``scrapy.utils.url.add_or_replace_parameter()`` + + - ``scrapy.utils.url.add_or_replace_parameters()`` + + - ``scrapy.utils.url.any_to_uri()`` + + - ``scrapy.utils.url.canonicalize_url()`` + + - ``scrapy.utils.url.file_uri_to_path()`` + + - ``scrapy.utils.url.is_url()`` + + - ``scrapy.utils.url.parse_data_uri()`` + + - ``scrapy.utils.url.parse_url()`` + + - ``scrapy.utils.url.path_to_file_uri()`` + + - ``scrapy.utils.url.safe_download_url()`` + + - ``scrapy.utils.url.safe_url_string()`` + + - ``scrapy.utils.url.url_query_cleaner()`` + + - ``scrapy.utils.url.url_query_parameter()`` + + (:issue:`4577`, :issue:`6583`, :issue:`6586`) + +- HTTP/1.0 support code is deprecated. It was disabled by default and + couldn't be used together with HTTP/1.1. If you still need it, you should + write your own download handler or copy the code from Scrapy. The + deprecations include: + + - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler`` + + - ``scrapy.core.downloader.webclient.ScrapyHTTPClientFactory`` + + - ``scrapy.core.downloader.webclient.ScrapyHTTPPageGetter`` + + - Overriding + ``scrapy.core.downloader.contextfactory.ScrapyClientContextFactory.getContext()`` + + (:issue:`6634`) + +- The following modules and functions used only in tests are deprecated: + + - the ``scrapy.utils.testproc`` module + + - the ``scrapy.utils.testsite`` module + + - ``scrapy.utils.test.assert_gcs_environ()`` + + - ``scrapy.utils.test.get_ftp_content_and_delete()`` + + - ``scrapy.utils.test.get_gcs_content_and_delete()`` + + - ``scrapy.utils.test.mock_google_cloud_storage()`` + + - ``scrapy.utils.test.skip_if_no_boto()`` + + If you need to use them in your tests or code, you can copy the code from Scrapy. + (:issue:`6696`) + +- ``scrapy.utils.test.TestSpider`` is deprecated. If you need an empty spider + class you can use :class:`scrapy.utils.spider.DefaultSpider` or create your + own subclass of :class:`scrapy.Spider`. + (:issue:`6678`) + +- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` is + deprecated. It was disabled by default and isn't useful for most of the + existing websites. + (:issue:`6523`, :issue:`6651`, :issue:`6656`) + +- ``scrapy.utils.url.escape_ajax()`` is deprecated. + (:issue:`6523`, :issue:`6651`) + +- ``scrapy.spiders.init.InitSpider`` is deprecated. If you find it useful, + you can copy its code from Scrapy. + (:issue:`6708`, :issue:`6714`) + +- ``scrapy.utils.versions.scrapy_components_versions()`` is deprecated, use + :func:`scrapy.utils.versions.get_versions` instead. + (:issue:`6582`) + +- ``BaseDupeFilter.log()`` is deprecated. It does nothing and shouldn't be + called. + (:issue:`4151`) + +- Passing the ``spider`` argument to the following methods of + :class:`~scrapy.core.scraper.Scraper` is deprecated: + + - ``close_spider()`` + + - ``enqueue_scrape()`` + + - ``handle_spider_error()`` + + - ``handle_spider_output()`` + + (:issue:`6764`) + +New features +~~~~~~~~~~~~ + +- You can now yield the start requests and items of a spider from the + :meth:`~scrapy.Spider.start` spider method and from the + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` spider + middleware method, both :term:`asynchronous generators `. + + This makes it possible to use asynchronous code to generate those start + requests and items, e.g. reading them from a queue service or database + using an asynchronous client, without workarounds. + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- Start requests are now :ref:`scheduled ` as soon as + possible. + + As a result, their :attr:`~scrapy.Request.priority` is now taken into + account as soon as :setting:`CONCURRENT_REQUESTS` is reached. + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- :class:`Crawler.signals ` has a new + :meth:`~scrapy.signalmanager.SignalManager.wait_for` method. + (:issue:`6729`) + +- Added a new :signal:`scheduler_empty` signal. + (:issue:`6729`) + +- Added new settings: :setting:`SCHEDULER_START_DISK_QUEUE` and + :setting:`SCHEDULER_START_MEMORY_QUEUE`. + (:issue:`6729`) + +- Added :class:`~scrapy.spidermiddlewares.start.StartSpiderMiddleware`, which + sets :reqmeta:`is_start_request` to ``True`` on :ref:`start requests + `. + (:issue:`6729`) + +- Exposed a new method of :class:`Crawler.engine + `: + :meth:`~scrapy.core.engine.ExecutionEngine.needs_backout`. + (:issue:`6729`) + +- Added the :reqmeta:`allow_offsite` request meta key that can be used + instead of the more general :attr:`~scrapy.Request.dont_filter` request + attribute to skip processing of the request by + :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` (but not + by other code that checks :attr:`~scrapy.Request.dont_filter`). + (:issue:`3690`, :issue:`6151`, :issue:`6366`) + +- Added an optional base class for spider middlewares, + :class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which can be + helpful for writing :ref:`universal spider middlewares + ` without boilerplate and code duplication. + The built-in spider middlewares now inherit from this class. + (:issue:`6693`, :issue:`6777`) + +- :ref:`Scrapy add-ons ` can now define a class method called + ``update_pre_crawler_settings()`` to update :ref:`pre-crawler settings + `. + (:issue:`6544`, :issue:`6568`) + +- Added :ref:`helpers ` for modifying :ref:`component + priority dictionary ` settings. + (:issue:`6614`) + +- Responses that use an unknown/unsupported encoding now produce a warning. + If Scrapy knows that installing an additional package (such as brotli_) + will allow decoding the response, that will be mentioned in the warning. + (:issue:`4697`, :issue:`6618`) + +- Added the ``spider_exceptions/count`` stat which tracks the total count of + exceptions (tracked also by per-type ``spider_exceptions/*`` stats). + (:issue:`6739`, :issue:`6740`) + +- Added the :setting:`DEFAULT_DROPITEM_LOG_LEVEL` setting and the + :attr:`scrapy.exceptions.DropItem.log_level` attribute that allow + customizing the log level of the message that is logged when an item is + dropped. + (:issue:`6603`, :issue:`6608`) + +- Added support for the ``-b, --cookie`` curl argument to + :meth:`scrapy.Request.from_curl`. + (:issue:`6684`) + +- Added the :setting:`LOG_VERSIONS` setting that allows customizing the + list of software whose versions are logged when the spider starts. + (:issue:`6582`) + +- Added the :setting:`WARN_ON_GENERATOR_RETURN_VALUE` setting that allows + disabling run time analysis of callback code used to warn about incorrect + ``return`` statements in generator-based callbacks. You may need to disable + this setting if this analysis breaks on your callback code. + (:issue:`6731`, :issue:`6738`) + +Improvements +~~~~~~~~~~~~ + +- Removed or postponed some calls of :func:`itemadapter.is_item` to increase + performance. + (:issue:`6719`) + +- Improved the error message when running a ``scrapy`` command that requires + a project (such as ``scrapy crawl``) outside of a project directory. + (:issue:`2349`, :issue:`3426`) + +- Added an empty :setting:`ADDONS` setting to the ``settings.py`` template + for new projects. + (:issue:`6587`) + +Bug fixes +~~~~~~~~~ + +- Yielding an item from :meth:`Spider.start ` or from + :meth:`SpiderMiddleware.process_start + ` no longer delays + the next iteration of starting requests and items by up to 5 seconds. + (:issue:`6729`) + +- Fixed calculation of ``items_per_minute`` and ``responses_per_minute`` + stats. + (:issue:`6599`) + +- Fixed an error initializing + :class:`scrapy.extensions.feedexport.GCSFeedStorage`. + (:issue:`6617`, :issue:`6628`) + +- Fixed an error running ``scrapy bench``. + (:issue:`6632`, :issue:`6633`) + +- Fixed duplicated log messages about the reactor and the event loop. + (:issue:`6636`, :issue:`6657`) + +- Fixed resolving type annotations of ``SitemapSpider._parse_sitemap()`` at + run time, required by tools such as scrapy-poet_. + (:issue:`6665`, :issue:`6671`) + + .. _scrapy-poet: https://github.com/scrapinghub/scrapy-poet + +- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without + an installed reactor now raises an exception instead of installing a + reactor. + (:issue:`6732`, :issue:`6735`) + +- Restored support for the ``x-gzip`` content encoding. + (:issue:`6618`) + +Documentation +~~~~~~~~~~~~~ + +- Documented the setting values set in the default project template. + (:issue:`6762`, :issue:`6775`) + +- Improved the docs about asynchronous iterable support in spider + middlewares. (:issue:`6688`) + +- Improved the :ref:`docs ` about using + :class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based + code and included a list of such APIs. + (:issue:`6677`, :issue:`6734`, :issue:`6776`) + +- Improved the :ref:`contribution docs `. + (:issue:`6561`, :issue:`6575`) + +- Removed the ``Splash`` recommendation from the :ref:`headless browser + ` suggestion. We no longer recommend using + ``Splash`` and recommend using other headless browser solutions instead. + (:issue:`6642`, :issue:`6701`) + +- Added the dark mode to the HTML documentation. + (:issue:`6653`) + +- Other documentation improvements and fixes. + (:issue:`4151`, + :issue:`6526`, + :issue:`6620`, + :issue:`6621`, + :issue:`6622`, + :issue:`6623`, + :issue:`6624`, + :issue:`6721`, + :issue:`6723`, + :issue:`6780`) + +Packaging +~~~~~~~~~ + +- Switched from ``setup.py`` to ``pyproject.toml``. + (:issue:`6514`, :issue:`6547`) + +- Switched the build backend from setuptools_ to hatchling_. + (:issue:`6771`) + + .. _hatchling: https://pypi.org/project/hatchling/ + +Quality assurance +~~~~~~~~~~~~~~~~~ + +- Replaced most linters with ruff_. + (:issue:`6565`, + :issue:`6576`, + :issue:`6577`, + :issue:`6581`, + :issue:`6584`, + :issue:`6595`, + :issue:`6601`, + :issue:`6631`) + + .. _ruff: https://docs.astral.sh/ruff/ + +- Improved accuracy and performance of collecting test coverage. + (:issue:`6255`, :issue:`6610`) + +- Fixed an error that prevented running tests from directories other than the + top level source directory. + (:issue:`6567`) + +- Reduced the amount of ``mockserver`` calls in tests to improve the overall + test run time. + (:issue:`6637`, :issue:`6648`) + +- Fixed tests that were running the same test code more than once. + (:issue:`6646`, :issue:`6647`, :issue:`6650`) + +- Refactored tests to use more ``pytest`` features instead of ``unittest`` + ones where possible. + (:issue:`6678`, + :issue:`6680`, + :issue:`6695`, + :issue:`6699`, + :issue:`6700`, + :issue:`6702`, + :issue:`6709`, + :issue:`6710`, + :issue:`6711`, + :issue:`6712`, + :issue:`6725`) + +- Type hints improvements and fixes. + (:issue:`6578`, + :issue:`6579`, + :issue:`6593`, + :issue:`6605`, + :issue:`6694`) + +- CI and test improvements and fixes. + (:issue:`5360`, + :issue:`6271`, + :issue:`6547`, + :issue:`6560`, + :issue:`6602`, + :issue:`6607`, + :issue:`6609`, + :issue:`6613`, + :issue:`6619`, + :issue:`6626`, + :issue:`6679`, + :issue:`6703`, + :issue:`6704`, + :issue:`6716`, + :issue:`6720`, + :issue:`6722`, + :issue:`6724`, + :issue:`6741`, + :issue:`6743`, + :issue:`6766`, + :issue:`6770`, + :issue:`6772`, + :issue:`6773`) + +- Code cleanups. + (:issue:`6600`, + :issue:`6606`, + :issue:`6635`, + :issue:`6764`) + + .. _release-2.12.0: Scrapy 2.12.0 (2024-11-18) @@ -12,7 +2514,7 @@ Highlights: - Dropped support for Python 3.8, added support for Python 3.13 -- :meth:`~scrapy.Spider.start_requests` can now yield items +- ``scrapy.Spider.start_requests()`` can now yield items - Added :class:`~scrapy.http.JsonResponse` @@ -44,7 +2546,7 @@ Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - User-defined cookies for HTTPS requests will have the ``secure`` flag set - to ``True`` unless it's set to ``False`` explictly. This is important when + to ``True`` unless it's set to ``False`` explicitly. This is important when these cookies are reused in HTTP requests, e.g. after a redirect to an HTTP URL. (:issue:`6357`) @@ -79,7 +2581,7 @@ Backward-incompatible changes ``crawler.settings`` instead. When they call ``__init__()`` of the base class they should pass the ``crawler`` argument to it too. - A ``from_settings()`` method shouldn't be defined. Class-specific - initialization code should go into either an overriden ``from_crawler()`` + initialization code should go into either an overridden ``from_crawler()`` method or into ``__init__()``. - It's now possible to override ``from_crawler()`` and it's not necessary to call ``MediaPipeline.from_crawler()`` in it if other recommendations @@ -163,7 +2665,7 @@ Deprecation removals (:issue:`6109`, :issue:`6116`) - A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that - does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface + does not implement the ``ISpiderLoader`` interface will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at run time. Non-compliant classes have been triggering a deprecation warning since Scrapy 1.0.0. @@ -303,9 +2805,13 @@ Deprecations New features ~~~~~~~~~~~~ -- :meth:`~scrapy.Spider.start_requests` can now yield items. +- ``scrapy.Spider.start_requests()`` can now yield items. (:issue:`5289`, :issue:`6417`) + .. note:: Some spider middlewares may need to be updated for Scrapy 2.12 + support before you can use them in combination with the ability to + yield items from ``start_requests()``. + - Added a new :class:`~scrapy.http.Response` subclass, :class:`~scrapy.http.JsonResponse`, for responses with a `JSON MIME type `_. @@ -595,14 +3101,23 @@ Security bug fixes .. _defusedxml: https://github.com/tiran/defusedxml +Deprecations +~~~~~~~~~~~~ + +- ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware`` (a spider + middleware) is now deprecated and not enabled by default. The new + downloader middleware with the same functionality, + :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`, is enabled + instead. + (:issue:`2241`, :issue:`6358`) + + Bug fixes ~~~~~~~~~ - Restored support for brotlipy_, which had been dropped in Scrapy 2.11.1 in favor of brotli_. (:issue:`6261`) - .. _brotli: https://github.com/google/brotli - .. note:: brotlipy is deprecated, both in Scrapy and upstream. Use brotli instead if you can. @@ -795,7 +3310,7 @@ Backward-incompatible changes in :meth:`scrapy.Spider.from_crawler`. If you want to access the final setting values and the initialized :class:`~scrapy.crawler.Crawler` attributes in the spider code as early as possible you can do this in - :meth:`~scrapy.Spider.start_requests` or in a handler of the + ``scrapy.Spider.start_requests()`` or in a handler of the :signal:`engine_started` signal. (:issue:`6038`) - The :meth:`TextResponse.json ` method now @@ -934,10 +3449,10 @@ Modified requirements Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` - instead of ``False``. In earlier Scrapy versions empty files were created - even when this setting was ``False`` (which was a bug that is now fixed), - so the new default should keep the old behavior. (:issue:`872`, +- The value of the :setting:`FEED_STORE_EMPTY` setting is now ``True`` + instead of ``False``. In earlier Scrapy versions empty files were created + even when this setting was ``False`` (which was a bug that is now fixed), + so the new default should keep the old behavior. (:issue:`872`, :issue:`5847`) Deprecation removals @@ -1150,7 +3665,7 @@ Bug fixes (:issue:`5914`, :issue:`5917`) - Fixed an error breaking user handling of send failures in - :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`) + :meth:`scrapy.mail.MailSender.send`. (:issue:`1611`, :issue:`5880`) Documentation ~~~~~~~~~~~~~ @@ -1569,7 +4084,7 @@ Bug fixes that does not match the asyncio event loop actually installed (:issue:`5529`). -- Fixed :meth:`Headers.getlist ` +- Fixed :meth:`Headers.getlist() ` returning only the last header (:issue:`5515`, :issue:`5526`). - Fixed :class:`LinkExtractor @@ -1814,7 +4329,7 @@ Modified requirements ~~~~~~~~~~~~~~~~~~~~~ - The h2_ dependency is now optional, only needed to - :ref:`enable HTTP/2 support `. (:issue:`5113`) + :ref:`enable HTTP/2 support `. (:issue:`5113`) .. _h2: https://pypi.org/project/h2/ @@ -2186,8 +4701,6 @@ Scrapy 2.5.1 (2021-10-05) need to upgrade scrapy-splash to a greater version for it to continue to work. -.. _scrapy-splash: https://github.com/scrapy-plugins/scrapy-splash - .. _release-2.5.0: @@ -2198,7 +4711,7 @@ Highlights: - Official Python 3.9 support -- Experimental :ref:`HTTP/2 support ` +- Experimental :ref:`HTTP/2 support ` - New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function to retry requests from spider callbacks @@ -2229,7 +4742,7 @@ Deprecations New features ~~~~~~~~~~~~ -- Experimental :ref:`HTTP/2 support ` through a new download handler +- Experimental :ref:`HTTP/2 support ` through a new download handler that can be assigned to the ``https`` protocol in the :setting:`DOWNLOAD_HANDLERS` setting. (:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`) @@ -2642,7 +5155,7 @@ Bug fixes * The system file mode creation mask no longer affects the permissions of files generated using the :command:`startproject` command (:issue:`4722`) -* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names +* ``scrapy.utils.iterators.xmliter`` now supports namespaced node names (:issue:`861`, :issue:`4746`) * :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can @@ -3203,7 +5716,7 @@ Highlights: * :ref:`FTP support ` for media pipelines * New :attr:`Response.certificate ` attribute -* IPv6 support through :setting:`DNS_RESOLVER` +* IPv6 support through ``DNS_RESOLVER`` Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -3311,7 +5824,7 @@ New features :class:`twisted.internet.ssl.Certificate` object for HTTPS responses (:issue:`2726`, :issue:`4054`) -* A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support +* A new ``DNS_RESOLVER`` setting allows enabling IPv6 support (:issue:`1031`, :issue:`4227`) * A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring @@ -3371,7 +5884,7 @@ New features * :class:`~scrapy.spiders.Spider` objects now raise an :exc:`AttributeError` exception if they do not have a :class:`~scrapy.spiders.Spider.start_urls` - attribute nor reimplement :class:`~scrapy.spiders.Spider.start_requests`, + attribute nor reimplement ``scrapy.spiders.Spider.start_requests()``, but have a ``start_url`` attribute (:issue:`4133`, :issue:`4170`) * :class:`~scrapy.exporters.BaseItemExporter` subclasses may now use @@ -3391,7 +5904,7 @@ New features components already supported (:issue:`4126`) * :class:`scrapy.utils.python.MutableChain.__iter__` now returns ``self``, - `allowing it to be used as a sequence `_ + allowing it to be used as a sequence. (:issue:`4153`) @@ -3555,7 +6068,7 @@ The following changes may impact custom priority queue classes: * A new keyword parameter has been added: ``key``. It is a string that is always an empty string for memory queues and indicates the - :setting:`JOB_DIR` value for disk queues. + :setting:`JOBDIR` value for disk queues. * The parameter for disk queues that contains data from the previous crawl, ``startprios`` or ``slot_startprios``, is now passed as a @@ -3820,6 +6333,8 @@ Highlights: Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. skip: start + * Python 3.4 is no longer supported, and some of the minimum requirements of Scrapy have also changed: @@ -3839,7 +6354,7 @@ Backward-incompatible changes consistency with similar classes (:issue:`3929`, :issue:`3982`) * If you are using a custom context factory - (:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`), its ``__init__`` method must + (``DOWNLOADER_CLIENTCONTEXTFACTORY``), its ``__init__`` method must accept two new parameters: ``tls_verbose_logging`` and ``tls_ciphers`` (:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`) @@ -3860,6 +6375,8 @@ Backward-incompatible changes (:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`, :issue:`3998`, :issue:`4036`) +.. skip: end + See also :ref:`1.8-deprecation-removals` below. @@ -4072,8 +6589,8 @@ Backward-incompatible changes ``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly. * :class:`~scrapy.crawler.Crawler`, - :class:`CrawlerRunner.crawl ` and - :class:`CrawlerRunner.create_crawler ` + :meth:`CrawlerRunner.crawl ` and + :meth:`CrawlerRunner.create_crawler ` no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they only accept a :class:`~scrapy.spiders.Spider` subclass now. @@ -4105,7 +6622,7 @@ New features ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be :ref:`enabled ` for a significant scheduling improvement on crawls targeting multiple web domains, at the - cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`) + cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`) * A new :attr:`.Request.cb_kwargs` attribute provides a cleaner way to pass keyword arguments to callback methods @@ -4698,7 +7215,7 @@ Docs - Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY`` setting. (:issue:`2756`) - Update Contributing docs, document new support channels - (:issue:`2762`, issue:`3038`) + (:issue:`2762`, :issue:`3038`) - Include references to Scrapy subreddit in the docs - Fix broken links; use ``https://`` for external links (:issue:`2978`, :issue:`2982`, :issue:`2958`) @@ -5293,7 +7810,7 @@ This 1.1 release brings a lot of interesting features and bug fixes: selectors engine without needing to upgrade Scrapy. - HTTPS downloader now does TLS protocol negotiation by default, instead of forcing TLS 1.0. You can also set the SSL/TLS method - using the new :setting:`DOWNLOADER_CLIENT_TLS_METHOD`. + using the new ``DOWNLOADER_CLIENT_TLS_METHOD`` setting. - These bug fixes may require your attention: @@ -5328,8 +7845,7 @@ Keep reading for more details on other improvements and bug fixes. Beta Python 3 Support ~~~~~~~~~~~~~~~~~~~~~ -We have been `hard at work to make Scrapy run on Python 3 -`_. As a result, now +We have been hard at work to make Scrapy run on Python 3. As a result, now you can run spiders on Python 3.3, 3.4 and 3.5 (Twisted >= 15.5 required). Some features are still missing (and some may never be ported). @@ -5397,7 +7913,7 @@ Additional New Features and Enhancements - Other refactoring, optimizations and cleanup (:issue:`1476`, :issue:`1481`, :issue:`1477`, :issue:`1315`, :issue:`1290`, :issue:`1750`, :issue:`1881`). -.. _`Code of Conduct`: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md +.. _Code of Conduct: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md Deprecations and Removals @@ -6159,7 +8675,7 @@ Enhancements - Make ``RFPDupeFilter`` class easily subclassable (:issue:`533`) - Improve test coverage and forthcoming Python 3 support (:issue:`525`) - Promote startup info on settings and middleware to INFO level (:issue:`520`) -- Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`) +- Support partials in ``get_func_args`` util (:issue:`506`, :issue:`504`) - Allow running individual tests via tox (:issue:`503`) - Update extensions ignored by link extractors (:issue:`498`) - Add middleware methods to get files/images/thumbs paths (:issue:`490`) @@ -6292,7 +8808,7 @@ Scrapy 0.18.4 (released 2013-10-10) - IPython refuses to update the namespace. fix #396 (:commit:`3d32c4f`) - Fix AlreadyCalledError replacing a request in shell command. closes #407 (:commit:`b1d8919`) -- Fix start_requests laziness and early hangs (:commit:`89faf52`) +- Fix ``start_requests()`` laziness and early hangs (:commit:`89faf52`) Scrapy 0.18.3 (released 2013-10-03) ----------------------------------- @@ -6485,7 +9001,7 @@ Scrapy changes: - added options ``-o`` and ``-t`` to the :command:`runspider` command - documented :doc:`topics/autothrottle` and added to extensions installed by default. You still need to enable it with :setting:`AUTOTHROTTLE_ENABLED` - major Stats Collection refactoring: removed separation of global/per-spider stats, removed stats-related signals (``stats_spider_opened``, etc). Stats are much simpler now, backward compatibility is kept on the Stats Collector API and signals. -- added :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests` method to spider middlewares +- added a ``process_start_requests()`` method to spider middlewares - dropped Signals singleton. Signals should now be accessed through the Crawler.signals attribute. See the signals documentation for more info. - dropped Stats Collector singleton. Stats can now be accessed through the Crawler.stats attribute. See the stats collection documentation for more info. - documented :ref:`topics-api` @@ -6548,7 +9064,7 @@ Scrapy 0.14.2 - fixed bug in MemoryUsage extension: get_engine_status() takes exactly 1 argument (0 given) (:commit:`11133e9`) - fixed struct.error on http compression middleware. closes #87 (:commit:`1423140`) - ajax crawling wasn't expanding for unicode urls (:commit:`0de3fb4`) -- Catch start_requests iterator errors. refs #83 (:commit:`454a21d`) +- Catch ``start_requests()`` iterator errors. refs #83 (:commit:`454a21d`) - Speed-up libxml2 XPathSelector (:commit:`2fbd662`) - updated versioning doc according to recent changes (:commit:`0a070f5`) - scrapyd: fixed documentation link (:commit:`2b4e4c3`) @@ -6585,7 +9101,7 @@ New features and settings - In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`) - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`) - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by: - - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP` + - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP`` - check the documentation for more details - Added builtin caching DNS resolver (:rev:`2728`) - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`) @@ -6727,7 +9243,7 @@ API changes - ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler`` - ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler`` - Moved module: ``scrapy.contrib.spidermanager`` to ``scrapy.spidermanager`` -- Spider Manager singleton moved from ``scrapy.spider.spiders`` to the ``spiders` attribute of ``scrapy.project.crawler`` singleton. +- Spider Manager singleton moved from ``scrapy.spider.spiders`` to the ``spiders`` attribute of ``scrapy.project.crawler`` singleton. - moved Stats Collector classes: (#204) - ``scrapy.stats.collector.StatsCollector`` to ``scrapy.statscol.StatsCollector`` - ``scrapy.stats.collector.SimpledbStatsCollector`` to ``scrapy.contrib.statscol.SimpledbStatsCollector`` @@ -6862,7 +9378,7 @@ First release of Scrapy. .. _parsel.csstranslator.HTMLTranslator: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.HTMLTranslator .. _parsel.csstranslator.XPathExpr: https://parsel.readthedocs.io/en/latest/parsel.html#parsel.csstranslator.XPathExpr .. _PEP 257: https://peps.python.org/pep-0257/ -.. _Pillow: https://python-pillow.org/ +.. _Pillow: https://github.com/python-pillow/Pillow .. _pyOpenSSL: https://www.pyopenssl.org/en/stable/ .. _queuelib: https://github.com/scrapy/queuelib .. _registered with IANA: https://www.iana.org/assignments/media-types/media-types.xhtml diff --git a/docs/requirements.in b/docs/requirements.in new file mode 100644 index 000000000..a1f3a7468 --- /dev/null +++ b/docs/requirements.in @@ -0,0 +1,8 @@ +h2 +pydantic +scrapy-spider-metadata +sphinx +sphinx-notfound-page +sphinx-rtd-theme +sphinx-rtd-dark-mode +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8 diff --git a/docs/requirements.txt b/docs/requirements.txt index 5f683d34c..a5cbad302 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,197 @@ -sphinx==6.2.1 -sphinx-hoverxref==1.3.0 -sphinx-notfound-page==1.0.0 -sphinx-rtd-theme==2.0.0 +# This file was autogenerated by uv via the following command: +# uv pip compile -p 3.13 docs/requirements.in -o docs/requirements.txt +alabaster==1.0.0 + # via sphinx +annotated-types==0.7.0 + # via pydantic +attrs==26.1.0 + # via + # service-identity + # twisted +automat==25.4.16 + # via twisted +babel==2.18.0 + # via sphinx +certifi==2026.2.25 + # via requests +cffi==2.0.0 + # via cryptography +charset-normalizer==3.4.6 + # via requests +constantly==23.10.4 + # via twisted +cryptography==46.0.6 + # via + # pyopenssl + # scrapy + # service-identity +cssselect==1.4.0 + # via + # parsel + # scrapy +defusedxml==0.7.1 + # via scrapy +docutils==0.22.4 + # via + # sphinx + # sphinx-markdown-builder + # sphinx-rtd-theme +filelock==3.25.2 + # via tldextract +h2==4.3.0 + # via -r docs/requirements.in +hpack==4.1.0 + # via h2 +hyperframe==6.1.0 + # via h2 +hyperlink==21.0.0 + # via twisted +idna==3.11 + # via + # hyperlink + # requests + # tldextract +imagesize==2.0.0 + # via sphinx +incremental==24.11.0 + # via twisted +itemadapter==0.13.1 + # via + # itemloaders + # scrapy +itemloaders==1.4.0 + # via scrapy +jinja2==3.1.6 + # via sphinx +jmespath==1.1.0 + # via + # itemloaders + # parsel +lxml==6.0.2 + # via + # parsel + # scrapy +markupsafe==3.0.3 + # via jinja2 +packaging==26.0 + # via + # incremental + # parsel + # scrapy + # scrapy-spider-metadata + # sphinx + # sphinx-scrapy +parsel==1.11.0 + # via + # itemloaders + # scrapy +protego==0.6.0 + # via scrapy +pyasn1==0.6.3 + # via + # pyasn1-modules + # service-identity +pyasn1-modules==0.4.2 + # via service-identity +pycparser==3.0 + # via cffi +pydantic==2.12.5 + # via + # -r docs/requirements.in + # scrapy-spider-metadata +pydantic-core==2.41.5 + # via pydantic +pydispatcher==2.0.7 + # via scrapy +pygments==2.19.2 + # via sphinx +pyopenssl==26.0.0 + # via scrapy +queuelib==1.9.0 + # via scrapy +requests==2.33.0 + # via + # requests-file + # sphinx + # tldextract +requests-file==3.0.1 + # via tldextract +roman-numerals==4.1.0 + # via sphinx +scrapy==2.14.2 + # via scrapy-spider-metadata +scrapy-spider-metadata==0.2.0 + # via -r docs/requirements.in +service-identity==24.2.0 + # via scrapy +snowballstemmer==3.0.1 + # via sphinx +sphinx==9.1.0 + # via + # -r docs/requirements.in + # sphinx-copybutton + # sphinx-last-updated-by-git + # sphinx-llms-txt + # sphinx-markdown-builder + # sphinx-notfound-page + # sphinx-rtd-theme + # sphinx-scrapy + # sphinxcontrib-jquery +sphinx-copybutton==0.5.2 + # via sphinx-scrapy +sphinx-last-updated-by-git==0.3.8 + # via sphinx-sitemap +sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69 + # via sphinx-scrapy +sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@cfe4c0bfd7b4542f7e6b65a58cdf9ec765829940 + # via sphinx-scrapy +sphinx-notfound-page==1.1.0 + # via -r docs/requirements.in +sphinx-rtd-dark-mode==1.3.0 + # via -r docs/requirements.in +sphinx-rtd-theme==3.1.0 + # via + # -r docs/requirements.in + # sphinx-rtd-dark-mode +sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737 + # via -r docs/requirements.in +sphinx-sitemap==2.9.0 + # via sphinx-scrapy +sphinxcontrib-applehelp==2.0.0 + # via sphinx +sphinxcontrib-devhelp==2.0.0 + # via sphinx +sphinxcontrib-htmlhelp==2.1.0 + # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme +sphinxcontrib-jsmath==1.0.1 + # via sphinx +sphinxcontrib-qthelp==2.0.0 + # via sphinx +sphinxcontrib-serializinghtml==2.0.0 + # via sphinx +tabulate==0.10.0 + # via sphinx-markdown-builder +tldextract==5.3.1 + # via scrapy +twisted==25.5.0 + # via scrapy +typing-extensions==4.15.0 + # via + # pydantic + # pydantic-core + # twisted + # typing-inspection +typing-inspection==0.4.2 + # via pydantic +urllib3==2.6.3 + # via requests +w3lib==2.4.1 + # via + # parsel + # scrapy +zope-interface==8.2 + # via + # scrapy + # twisted diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst index 14b4aa8ba..75f15b3ae 100644 --- a/docs/topics/addons.rst +++ b/docs/topics/addons.rst @@ -21,10 +21,14 @@ The ``ADDONS`` setting is a dict in which every key is an add-on class or its import path and the value is its priority. This is an example where two add-ons are enabled in a project's -``settings.py``:: +``settings.py``: + +.. skip: next + +.. code-block:: python ADDONS = { - 'path.to.someaddon': 0, + "path.to.someaddon": 0, SomeAddonClass: 1, } @@ -32,7 +36,8 @@ This is an example where two add-ons are enabled in a project's Writing your own add-ons ======================== -Add-ons are Python classes that include the following method: +Add-ons are :ref:`components ` that include one or both of +the following methods: .. method:: update_settings(settings) @@ -45,37 +50,30 @@ Add-ons are Python classes that include the following method: :param settings: The settings object storing Scrapy/component configuration :type settings: :class:`~scrapy.settings.Settings` -They can also have the following method: +.. classmethod:: update_pre_crawler_settings(cls, settings) -.. classmethod:: from_crawler(cls, crawler) - :noindex: + Use this class method instead of the :meth:`update_settings` method to + update :ref:`pre-crawler settings ` whose value is + used before the :class:`~scrapy.crawler.Crawler` object is created. - If present, this class method is called to create an add-on instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the add-on. The crawler object provides access to all Scrapy core - components like settings and signals; it is a way for the add-on to access - them and hook its functionality into Scrapy. - - :param crawler: The crawler that uses this add-on - :type crawler: :class:`~scrapy.crawler.Crawler` + :param settings: The settings object storing Scrapy/component configuration + :type settings: :class:`~scrapy.settings.BaseSettings` The settings set by the add-on should use the ``addon`` priority (see -:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):: +:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`): + +.. code-block:: python class MyAddon: def update_settings(self, settings): settings.set("DNSCACHE_ENABLED", True, "addon") This allows users to override these settings in the project or spider -configuration. This is not possible with settings that are mutable objects, -such as the dict that is a value of :setting:`ITEM_PIPELINES`. In these cases -you can provide an add-on-specific setting that governs whether the add-on will -modify :setting:`ITEM_PIPELINES`:: +configuration. - class MyAddon: - def update_settings(self, settings): - if settings.getbool("MYADDON_ENABLE_PIPELINE"): - settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 +When editing the value of a setting instead of overriding it entirely, it is +usually best to leave its priority unchanged. For example, when editing a +:ref:`component priority dictionary `. If the ``update_settings`` method raises :exc:`scrapy.exceptions.NotConfigured`, the add-on will be skipped. This makes @@ -96,7 +94,7 @@ recommend that such custom components should be written in the following way: 1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the default Scrapy one (e.g. - ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead + ``scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler``), but instead be able to load the class of the fallback component from a special setting (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use it. @@ -106,9 +104,9 @@ recommend that such custom components should be written in the following way: (``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default setting to the component provided by the add-on (e.g. ``MyDownloadHandler``). If the fallback setting is already set by the user, - they shouldn't change it. + it should not be changed. 3. This way, if there are several add-ons that want to modify the same setting, - all of them will fallback to the component from the previous one and then to + all of them will fall back to the component from the previous one and then to the Scrapy default. The order of that depends on the priority order in the ``ADDONS`` setting. @@ -118,12 +116,30 @@ Add-on examples Set some basic configuration: +.. skip: next .. code-block:: python + from myproject.pipelines import MyPipeline + + class MyAddon: def update_settings(self, settings): - settings["ITEM_PIPELINES"]["path.to.mypipeline"] = 200 settings.set("DNSCACHE_ENABLED", True, "addon") + settings.remove_from_list("METAREFRESH_IGNORE_TAGS", "noscript") + settings.setdefault_in_component_priority_dict( + "ITEM_PIPELINES", MyPipeline, 200 + ) + +.. _priority-dict-helpers: + +.. tip:: When editing a :ref:`component priority dictionary + ` setting, like :setting:`ITEM_PIPELINES`, + consider using setting methods like + :meth:`~scrapy.settings.BaseSettings.replace_in_component_priority_dict`, + :meth:`~scrapy.settings.BaseSettings.set_in_component_priority_dict` + and + :meth:`~scrapy.settings.BaseSettings.setdefault_in_component_priority_dict` + to avoid mistakes. Check dependencies: @@ -156,9 +172,7 @@ Use a fallback component: .. code-block:: python - from scrapy.core.downloader.handlers.http import HTTPDownloadHandler - from scrapy.utils.misc import build_from_crawler - + from scrapy.utils.misc import build_from_crawler, load_object FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" @@ -166,16 +180,19 @@ Use a fallback component: class MyHandler: lazy = False - def __init__(self, settings, crawler): - dhcls = load_object(settings.get(FALLBACK_SETTING)) + def __init__(self, crawler): + dhcls = load_object(crawler.settings.get(FALLBACK_SETTING)) self._fallback_handler = build_from_crawler(dhcls, crawler) - def download_request(self, request, spider): + async def download_request(self, request): if request.meta.get("my_params"): # handle the request ... else: - return self._fallback_handler.download_request(request, spider) + return await self._fallback_handler.download_request(request) + + async def close(self): + pass class MyAddon: diff --git a/docs/topics/api.rst b/docs/topics/api.rst index f7cffb61b..7ff8d3464 100644 --- a/docs/topics/api.rst +++ b/docs/topics/api.rst @@ -12,10 +12,11 @@ extensions and middlewares. Crawler API =========== -The main entry point to Scrapy API is the :class:`~scrapy.crawler.Crawler` -object, passed to extensions through the ``from_crawler`` class method. This -object provides access to all Scrapy core components, and it's the only way for -extensions to access them and hook their functionality into Scrapy. +The main entry point to the Scrapy API is the :class:`~scrapy.crawler.Crawler` +object, which :ref:`components ` can :ref:`get for +initialization `. It provides access to all Scrapy core +components, and it is the only way for components to access them and hook their +functionality into Scrapy. .. module:: scrapy.crawler :synopsis: The Scrapy crawler @@ -88,7 +89,7 @@ how you :ref:`configure the downloader middlewares The execution engine, which coordinates the core crawling logic between the scheduler, downloader and spiders. - Some extension may want to access the Scrapy engine, to inspect or + Some extension may want to access the Scrapy engine, to inspect or modify the downloader and scheduler behaviour, although this is an advanced use and this API is not yet stable. @@ -98,19 +99,25 @@ how you :ref:`configure the downloader middlewares provided while constructing the crawler, and it is created after the arguments given in the :meth:`crawl` method. - .. method:: crawl(*args, **kwargs) + .. automethod:: crawl_async - Starts the crawler by instantiating its spider class with the given - ``args`` and ``kwargs`` arguments, while setting the execution engine in - motion. Should be called only once. + .. automethod:: crawl - Returns a deferred that is fired when the crawl is finished. + .. automethod:: stop_async .. automethod:: stop +.. autoclass:: AsyncCrawlerRunner + :members: + .. autoclass:: CrawlerRunner :members: +.. autoclass:: AsyncCrawlerProcess + :show-inheritance: + :members: + :inherited-members: + .. autoclass:: CrawlerProcess :show-inheritance: :members: @@ -165,46 +172,17 @@ SpiderLoader API .. module:: scrapy.spiderloader :synopsis: The spider loader -.. class:: SpiderLoader +Custom spider loaders can be employed by specifying their path in the +:setting:`SPIDER_LOADER_CLASS` project setting. They must implement +:class:`SpiderLoaderProtocol`. - This class is in charge of retrieving and handling the spider classes - defined across the project. +.. autoclass:: SpiderLoaderProtocol + :members: - Custom spider loaders can be employed by specifying their path in the - :setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement - the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an - errorless execution. +.. autoclass:: SpiderLoader + :members: - .. method:: from_settings(settings) - - This class method is used by Scrapy to create an instance of the class. - It's called with the current project settings, and it loads the spiders - found recursively in the modules of the :setting:`SPIDER_MODULES` - setting. - - :param settings: project settings - :type settings: :class:`~scrapy.settings.Settings` instance - - .. method:: load(spider_name) - - Get the Spider class with the given name. It'll look into the previously - loaded spiders for a spider class with name ``spider_name`` and will raise - a KeyError if not found. - - :param spider_name: spider class name - :type spider_name: str - - .. method:: list() - - Get the names of the available spiders in the project. - - .. method:: find_by_request(request) - - List the spiders' names that can handle the given request. Will try to - match the request's url against the domains of the spiders. - - :param request: queried request - :type request: :class:`~scrapy.Request` instance +.. autoclass:: DummySpiderLoader .. _topics-api-signals: @@ -271,11 +249,17 @@ class (which they all inherit from). The following methods are not part of the stats collection api but instead used when implementing custom stats collectors: - .. method:: open_spider(spider) + .. method:: open_spider() - Open the given spider for stats collection. + Open the spider for stats collection. - .. method:: close_spider(spider) + .. method:: close_spider() - Close the given spider. After this is called, no more specific stats + Close the spider. After this is called, no more specific stats can be accessed or collected. + +Engine API +========== + +.. autoclass:: scrapy.core.engine.ExecutionEngine() + :members: needs_backout diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 0370dc538..c60c43f3c 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -63,7 +63,7 @@ this: :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`). 8. The :ref:`Engine ` sends processed items to - :ref:`Item Pipelines `, then send processed Requests to + :ref:`Item Pipelines `, then sends processed Requests to the :ref:`Scheduler ` and asks for possible next Requests to crawl. @@ -87,8 +87,8 @@ of the system, and triggering events when certain actions occur. See the Scheduler --------- -The :ref:`scheduler ` receives requests from the engine and -enqueues them for feeding them later (also to the engine) when the engine +The :ref:`scheduler ` receives requests from the engine and +enqueues them for feeding them later (also to the engine) when the engine requests them. .. _component-downloader: @@ -150,7 +150,7 @@ requests). Use a Spider middleware if you need to * post-process output of spider callbacks - change/add/remove requests or items; -* post-process start_requests; +* post-process start requests or items; * handle spider exceptions; * call errback instead of callback for some of the requests based on response content. diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 07baea071..63c217e93 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -4,27 +4,38 @@ asyncio ======= -.. versionadded:: 2.0 +Scrapy supports :mod:`asyncio` natively. New projects created with +:command:`startproject` have asyncio enabled by default, and you can use +:mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine +`. -Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the -asyncio reactor `, you may use :mod:`asyncio` and -:mod:`asyncio`-powered libraries in any :doc:`coroutine `. +The rest of this page covers advanced topics. If you are starting a new project, +no additional setup is needed. .. _install-asyncio: -Installing the asyncio reactor -============================== +Configuring the asyncio reactor +=============================== -To enable :mod:`asyncio` support, set the :setting:`TWISTED_REACTOR` setting to -``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``. +New projects generated with :command:`startproject` have the asyncio +reactor configured by default. No manual setup is needed. -If you are using :class:`~scrapy.crawler.CrawlerRunner`, you also need to +The :setting:`TWISTED_REACTOR` setting controls which Twisted reactor Scrapy +uses. Its default value is +``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, which enables +:mod:`asyncio` support. + +If you are using :class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.CrawlerRunner`, you also need to install the :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` reactor manually. You can do that using -:func:`~scrapy.utils.reactor.install_reactor`:: +:func:`~scrapy.utils.reactor.install_reactor`: - install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor') +.. skip: next +.. code-block:: python + + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") .. _asyncio-preinstalled-reactor: @@ -44,6 +55,7 @@ You can usually fix the issue by moving those offending module-level Twisted imports to the method or function definitions where they are used. For example, if you have something like: +.. skip: next .. code-block:: python from twisted.internet import reactor @@ -68,24 +80,36 @@ those imports happen. .. _asyncio-await-dfd: -Awaiting on Deferreds -===================== +Integrating Deferred code and asyncio code +========================================== -When the asyncio reactor isn't installed, you can await on Deferreds in the -coroutines directly. When it is installed, this is not possible anymore, due to -specifics of the Scrapy coroutine integration (the coroutines are wrapped into -:class:`asyncio.Future` objects, not into -:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into -Futures. Scrapy provides two helpers for this: +Coroutine functions can await on Deferreds by wrapping them into +:class:`asyncio.Future` objects. Scrapy provides two helpers for this: .. autofunction:: scrapy.utils.defer.deferred_to_future .. autofunction:: scrapy.utils.defer.maybe_deferred_to_future + +.. tip:: If you don't need to support reactors other than the default + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, you + can use :func:`~scrapy.utils.defer.deferred_to_future`, otherwise you + should use :func:`~scrapy.utils.defer.maybe_deferred_to_future`. + .. tip:: If you need to use these functions in code that aims to be compatible with lower versions of Scrapy that do not provide these functions, down to Scrapy 2.0 (earlier versions do not support :mod:`asyncio`), you can copy the implementation of these functions into your own code. +Coroutines and futures can be wrapped into Deferreds (for example, when a +Scrapy API requires passing a Deferred to it) using the following helpers: + +.. autofunction:: scrapy.utils.defer.deferred_from_coro +.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f + +The following function helps with a reverse wrapping: + +.. autofunction:: scrapy.utils.defer.ensure_awaitable + .. _enforce-asyncio-requirement: @@ -93,25 +117,202 @@ Enforcing asyncio as a requirement ================================== If you are writing a :ref:`component ` that requires asyncio -to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to +to work, use :func:`scrapy.utils.asyncio.is_asyncio_available` to :ref:`enforce it as a requirement `. For example: .. code-block:: python - from scrapy.utils.reactor import is_asyncio_reactor_installed + from scrapy.utils.asyncio import is_asyncio_available class MyComponent: def __init__(self): - if not is_asyncio_reactor_installed(): + if not is_asyncio_available(): raise ValueError( - f"{MyComponent.__qualname__} requires the asyncio Twisted " - f"reactor. Make sure you have it configured in the " + f"{MyComponent.__qualname__} requires the asyncio support. " + f"Make sure you have configured the asyncio reactor in the " f"TWISTED_REACTOR setting. See the asyncio documentation " f"of Scrapy for more information." ) +.. autofunction:: scrapy.utils.asyncio.is_asyncio_available +.. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed + + +.. _asyncio-without-reactor: + +Using Scrapy without a Twisted reactor +====================================== + +.. versionadded:: 2.15.0 + +.. warning:: + This is currently experimental and may not be suitable for production use. + +.. note:: As the Twisted download handlers cannot be used without a reactor, + the default download handler in this mode is + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. You + will need to additionally install the :ref:`httpx ` extra to use + it, unless you switch to some different handler. + +It's possible to use Scrapy without installing a Twisted reactor at all, by +setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this +mode Scrapy will use the asyncio event loop directly, and most of the Scrapy +functionality will work in the same way. + +Doing this provides several benefits in certain use cases: + +* A Twisted reactor, once stopped, cannot be started again. This prevents, for + example, using several instances of + :class:`~scrapy.crawler.AsyncCrawlerProcess` in the same process when they + use a reactor, but with ``TWISTED_REACTOR_ENABLED=False`` it becomes + possible. +* There may be limitations imposed by + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` and related + Twisted code, such as the requirement of using + :class:`~asyncio.SelectorEventLoop` on Windows (see :ref:`asyncio-windows`), + that do not apply if the reactor is not used. +* :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` manages the + underlying event loop, and while :class:`~scrapy.crawler.AsyncCrawlerRunner` + can use a pre-existing reactor which, in turn, can use a pre-existing event + loop, it's easier to use :class:`~scrapy.crawler.AsyncCrawlerRunner` with a + pre-existing loop directly. +* Omitting the reactor machinery may improve performance and reliability. + +Limitations +----------- + +As some Scrapy features and components require a reactor, they don't work and +are disabled without it. Replacements that don't require a reactor may be added +in future Scrapy versions. The following features are not available: + +* The default HTTP(S) download handler, + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` (this + is likely the biggest difference; Scrapy provides an HTTP(S) download handler + that doesn't require a reactor and will be used instead of it: + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`) +* :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` +* :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` +* :ref:`topics-telnetconsole` +* :class:`~scrapy.crawler.CrawlerRunner` and + :class:`~scrapy.crawler.CrawlerProcess` + (:class:`~scrapy.crawler.AsyncCrawlerProcess` and + :class:`~scrapy.crawler.AsyncCrawlerRunner` are available) +* Twisted-specific DNS resolvers (the :setting:`TWISTED_DNS_RESOLVER` setting) +* User and 3rd-party code that requires a reactor (see :ref:`below + ` for examples) + +Note that importing Twisted modules and, among other things, creating and using +:class:`~twisted.internet.defer.Deferred` objects doesn't require a reactor, so +code that uses :class:`~twisted.internet.defer.Deferred`, +:class:`~twisted.python.failure.Failure` and some other Twisted APIs will not +necessarily stop working. + +Other differences +----------------- + +When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``, Scrapy will change +the defaults of some other settings: + +* :setting:`TELNETCONSOLE_ENABLED` is set to ``False``. +* The ``"http"`` and ``"https"`` keys in :setting:`DOWNLOAD_HANDLERS_BASE` are + set to ``"scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler"``. +* The ``"ftp"`` key in :setting:`DOWNLOAD_HANDLERS_BASE` is set to ``None``. + +Thus, :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` is +used by default for making HTTP(S) requests. Please refer to its documentation +for its differences and limitations compared to +:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`. + +Additionally, :class:`~scrapy.crawler.AsyncCrawlerProcess` will install a +:term:`meta path finder` that prevents :mod:`twisted.internet.reactor` from +being imported. It will be uninstalled when :meth:`AsyncCrawlerProcess.start() +` exits. + +.. _asyncio-without-reactor-migrate: + +Adding support to existing code +------------------------------- + +Code that doesn't directly use Twisted APIs or APIs that depend on Twisted ones +doesn't need special support for running without a reactor. + +Here are some examples of APIs and patterns that need a replacement: + +* Using :meth:`reactor.callLater() + ` for sleeping or delayed calls. + You can use :meth:`asyncio.loop.call_later` instead. +* Using :func:`twisted.internet.threads.deferToThread`, + :meth:`reactor.callFromThread() + ` and related APIs to + execute code in other threads. You can use :func:`asyncio.to_thread`, + :meth:`asyncio.loop.call_soon_threadsafe` and related APIs instead. +* Using :class:`twisted.internet.task.LoopingCall` for scheduling repeated + tasks. As there is no direct replacement in the standard library, you may + need to write your own one using :func:`asyncio.sleep` in a task. +* Using Twisted network client and server APIs (:meth:`reactor.connectTCP() + `, + :meth:`reactor.listenTCP() + `, + :mod:`twisted.web.client`, :mod:`twisted.mail.smtp` etc.). You can use other + built-in or 3rd-party libraries for this. +* Using :class:`~scrapy.crawler.CrawlerProcess` or + :class:`~scrapy.crawler.CrawlerRunner`. You should use + :class:`~scrapy.crawler.AsyncCrawlerProcess` or + :class:`~scrapy.crawler.AsyncCrawlerRunner` respectively instead. +* Checking whether ``asyncio`` support is available with + :func:`scrapy.utils.reactor.is_asyncio_reactor_installed`. You should use + :func:`scrapy.utils.asyncio.is_asyncio_available` instead. + +Scrapy provides unified helpers for some of these examples: + +.. autofunction:: scrapy.utils.asyncio.call_later +.. autofunction:: scrapy.utils.asyncio.create_looping_call +.. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall +.. autofunction:: scrapy.utils.asyncio.run_in_thread + +If your code needs to know whether the reactor is available, you can either +check for the value of the :setting:`TWISTED_REACTOR_ENABLED` setting (you need +access to the :class:`~scrapy.crawler.Crawler` instance to do this) or use the +following function: + +.. autofunction:: scrapy.utils.reactorless.is_reactorless + +In general, code that doesn't use the reactor (directly or indirectly) can be +used unmodified both with the asyncio reactor and without a reactor. This +includes code that converts Deferreds to futures and vice versa as described in +:ref:`asyncio-await-dfd`. + +Troubleshooting +--------------- + +**ImportError: Import of twisted.internet.reactor is forbidden when running +without a Twisted reactor [...]:** Scrapy is configured to run without a +reactor, but some code imported :mod:`twisted.internet.reactor`, most likely +because that code needs a reactor to be used. You need to stop using this code +or set :setting:`TWISTED_REACTOR_ENABLED` back to ``True``. It's also possible +that the reactor isn't really needed but was installed due to the problem +described in :ref:`asyncio-preinstalled-reactor`, in which case it should be +enough to fix the problematic imports. + +**RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is +installed:** Scrapy is configured to run without a reactor, but a reactor is +already installed before the Scrapy code is executed. If you are trying to set +:setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings +`, it's currently unsupported. + +**RuntimeError: We expected a Twisted reactor to be installed but it isn't:** +Scrapy is configured to run with a reactor and not to install one, but a +reactor wasn't installed before the Scrapy code is executed. If you are trying +to set :setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings +`, it's currently unsupported. + +**RuntimeError: doesn't support TWISTED_REACTOR_ENABLED=False:** The +listed class cannot be used with :setting:`TWISTED_REACTOR_ENABLED` set to +``False``. There may be a replacement in the :ref:`documentation above +` or the documentation of the affected class. + .. _asyncio-windows: @@ -124,8 +325,7 @@ implementations, :class:`~asyncio.ProactorEventLoop` (default) and :class:`~asyncio.SelectorEventLoop` works with Twisted. Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop` -automatically when you change the :setting:`TWISTED_REACTOR` setting or call -:func:`~scrapy.utils.reactor.install_reactor`. +automatically when installing the asyncio reactor. .. note:: Other libraries you use may require :class:`~asyncio.ProactorEventLoop`, e.g. because it supports @@ -133,6 +333,9 @@ automatically when you change the :setting:`TWISTED_REACTOR` setting or call them together with Scrapy on Windows (but you should be able to use them on WSL or native Linux). +.. note:: This problem doesn't apply when not using the reactor, see + :ref:`asyncio-without-reactor`. + .. _playwright: https://github.com/microsoft/playwright-python @@ -144,3 +347,18 @@ Using custom asyncio loops You can also use custom asyncio event loops with the asyncio reactor. Set the :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to use it instead of the default asyncio event loop. + + +.. _disable-asyncio: + +Switching to a non-asyncio reactor +================================== + +If for some reason your code doesn't work with the asyncio reactor, you can use +a different reactor by setting the :setting:`TWISTED_REACTOR` setting to its +import path (e.g. ``'twisted.internet.epollreactor.EPollReactor'``) or to +``None``, which will use the default reactor for your platform. If you are +using :class:`~scrapy.crawler.AsyncCrawlerRunner` or +:class:`~scrapy.crawler.AsyncCrawlerProcess` you also need to switch to their +Deferred-based counterparts: :class:`~scrapy.crawler.CrawlerRunner` or +:class:`~scrapy.crawler.CrawlerProcess` respectively. diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst index 5bd72fa15..4f28019da 100644 --- a/docs/topics/autothrottle.rst +++ b/docs/topics/autothrottle.rst @@ -37,8 +37,7 @@ processed in parallel. Instead of adjusting the delays one can just set a small fixed download delay and impose hard limits on concurrency using -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP` options. It will provide a similar +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. It will provide a similar effect, but there are some important differences: * because the download delay is small there will be occasional bursts @@ -71,13 +70,12 @@ AutoThrottle algorithm adjusts download delays based on the following rules: .. note:: The AutoThrottle extension honours the standard Scrapy settings for concurrency and delay. This means that it will respect :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and - :setting:`CONCURRENT_REQUESTS_PER_IP` options and never set a download delay lower than :setting:`DOWNLOAD_DELAY`. .. _download-latency: In Scrapy, the download latency is measured as the time elapsed between -establishing the TCP connection and receiving the HTTP headers. +sending the request and receiving the HTTP headers. Note that these latencies are very hard to measure accurately in a cooperative multitasking environment because Scrapy may be busy processing a spider @@ -90,6 +88,8 @@ server) is, and this extension builds on that premise. Prevent specific requests from triggering slot delay adjustments ================================================================ +.. versionadded:: 2.12.0 + AutoThrottle adjusts the delay of download slots based on the latencies of responses that belong to that download slot. The only exceptions are non-200 responses, which are only taken into account to increase that delay, but @@ -123,7 +123,6 @@ The settings used to control the AutoThrottle extension are: * :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` * :setting:`AUTOTHROTTLE_DEBUG` * :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` -* :setting:`CONCURRENT_REQUESTS_PER_IP` * :setting:`DOWNLOAD_DELAY` For more information see :ref:`autothrottle-algorithm`. @@ -171,12 +170,10 @@ a higher value (e.g. ``2.0``) to increase the throughput and the load on remote servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value (e.g. ``0.5``) makes the crawler more conservative and polite. -Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` -and :setting:`CONCURRENT_REQUESTS_PER_IP` options are still respected +Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is still respected when AutoThrottle extension is enabled. This means that if ``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than -:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or -:setting:`CONCURRENT_REQUESTS_PER_IP`, the crawler won't reach this number +:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, the crawler won't reach this number of concurrent requests. At every given time point Scrapy can be sending more or less concurrent diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst index b704e54ed..e8ddec00c 100644 --- a/docs/topics/benchmarking.rst +++ b/docs/topics/benchmarking.rst @@ -83,4 +83,4 @@ and how well it's written. Use scrapy-bench_ for more complex benchmarking. -.. _scrapy-bench: https://github.com/scrapy/scrapy-bench \ No newline at end of file +.. _scrapy-bench: https://github.com/scrapy/scrapy-bench diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst index 0286c3354..cace1f883 100644 --- a/docs/topics/broad-crawls.rst +++ b/docs/topics/broad-crawls.rst @@ -41,19 +41,6 @@ efficient broad crawl. .. _broad-crawls-scheduler-priority-queue: -Use the right :setting:`SCHEDULER_PRIORITY_QUEUE` -================================================= - -Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``. -It works best during single-domain crawl. It does not work well with crawling -many different domains in parallel - -To apply the recommended priority queue use: - -.. code-block:: python - - SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue" - .. _broad-crawls-concurrency: Increase concurrency @@ -61,12 +48,7 @@ Increase concurrency Concurrency is the number of requests that are processed in parallel. There is a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that -can be set either per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) or per -IP (:setting:`CONCURRENT_REQUESTS_PER_IP`). - -.. note:: The scheduler priority queue :ref:`recommended for broad crawls - ` does not support - :setting:`CONCURRENT_REQUESTS_PER_IP`. +can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`). The default global concurrency limit in Scrapy is not suitable for crawling many different domains in parallel, so you will want to increase it. How much @@ -143,7 +125,7 @@ To disable cookies use: Disable retries =============== -Retrying failed HTTP requests can slow down the crawls substantially, specially +Retrying failed HTTP requests can slow down the crawls substantially, especially when sites causes are very slow (or fail) to respond, thus causing a timeout error which gets retried many times, unnecessarily, preventing crawler capacity to be reused for other domains. @@ -182,30 +164,6 @@ To disable redirects use: REDIRECT_ENABLED = False -Enable crawling of "Ajax Crawlable Pages" -========================================= - -Some pages (up to 1%, based on empirical data from year 2013) declare -themselves as ajax crawlable. This means they provide plain HTML -version of content that is usually available only via AJAX. -Pages can indicate it in two ways: - -1) by using ``#!`` in URL - this is the default way; -2) by using a special meta tag - this way is used on - "main", "index" website pages. - -Scrapy handles (1) automatically; to handle (2) enable -:ref:`AjaxCrawlMiddleware `: - -.. code-block:: python - - AJAXCRAWL_ENABLED = True - -When doing broad crawls it's common to crawl a lot of "index" web pages; -AjaxCrawlMiddleware helps to crawl them correctly. -It is turned OFF by default because it has some performance overhead, -and enabling it for focused crawls doesn't make much sense. - .. _broad-crawls-bfo: Crawl in BFO order diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 6ffb8ae93..ee2c3a3cd 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -163,8 +163,8 @@ information on which commands must be run from inside projects, and which not. Also keep in mind that some commands may have slightly different behaviours when running them from inside projects. For example, the fetch command will use -spider-overridden behaviours (such as the ``user_agent`` attribute to override -the user-agent) if the url being fetched is associated with some specific +spider-overridden behaviours (such as the ``custom_settings`` attribute to +override settings) if the url being fetched is associated with some specific spider. This is intentional, as the ``fetch`` command is meant to be used to check how spiders are downloading pages. @@ -199,6 +199,7 @@ Global commands: * :command:`fetch` * :command:`view` * :command:`version` +* :command:`bench` Project-only commands: @@ -207,7 +208,6 @@ Project-only commands: * :command:`list` * :command:`edit` * :command:`parse` -* :command:`bench` .. command:: startproject @@ -233,9 +233,6 @@ genspider * Syntax: ``scrapy genspider [-t template] `` * Requires project: *no* -.. versionadded:: 2.6.0 - The ability to pass a URL instead of a domain. - Creates a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes. Usage example:: @@ -312,11 +309,25 @@ Usage examples:: * parse_item $ scrapy check - [FAILED] first_spider:parse_item - >>> 'RetailPricex' field is missing + F.F. + ====================================================================== + FAIL: [first_spider] parse (@returns post-hook) + ---------------------------------------------------------------------- + Traceback (most recent call last): + ... + scrapy.exceptions.ContractFail: Returned 92 requests, expected 0..4 - [FAILED] first_spider:parse - >>> Returned 92 requests, expected 0..4 + ====================================================================== + FAIL: [first_spider] parse_item (@scrapes post-hook) + ---------------------------------------------------------------------- + Traceback (most recent call last): + ... + scrapy.exceptions.ContractFail: Missing fields: RetailPricex + + ---------------------------------------------------------------------- + Ran 4 contracts in 0.174s + + FAILED (failures=2) .. skip: end @@ -380,7 +391,7 @@ Supported options: * ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider -* ``--headers``: print the response's HTTP headers instead of the response's body +* ``--headers``: print the request's and response's HTTP headers instead of the response's body * ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) @@ -390,15 +401,19 @@ Usage examples:: [ ... html content here ... ] $ scrapy fetch --nolog --headers http://www.example.com/ - {'Accept-Ranges': ['bytes'], - 'Age': ['1263 '], - 'Connection': ['close '], - 'Content-Length': ['596'], - 'Content-Type': ['text/html; charset=UTF-8'], - 'Date': ['Wed, 18 Aug 2010 23:59:46 GMT'], - 'Etag': ['"573c1-254-48c9c87349680"'], - 'Last-Modified': ['Fri, 30 Jul 2010 15:30:18 GMT'], - 'Server': ['Apache/2.2.3 (CentOS)']} + > Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8 + > Accept-Language: en + > User-Agent: Scrapy/2.16.0 (+https://scrapy.org) + > Accept-Encoding: gzip, deflate, br + > + < Date: Wed, 08 Jul 2026 06:15:01 GMT + < Content-Type: text/html + < Server: cloudflare + < Last-Modified: Wed, 01 Jul 2026 17:50:18 GMT + < Allow: GET, HEAD + < Cf-Cache-Status: HIT + < Age: 8184 + < Cf-Ray: a17cf3b80eddf141-DME .. command:: view @@ -479,7 +494,7 @@ Supported options: * ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider -* ``--a NAME=VALUE``: set spider argument (may be repeated) +* ``-a NAME=VALUE``: set spider argument (may be repeated) * ``--callback`` or ``-c``: spider method to use as callback for parsing the response @@ -509,8 +524,6 @@ Supported options: * ``--output`` or ``-o``: dump scraped items to a file - .. versionadded:: 2.3 - .. skip: start Usage example:: @@ -587,6 +600,47 @@ bench Run a quick benchmark test. :ref:`benchmarking`. +.. _topics-commands-crawlerprocess: + +Commands that run a crawl +========================= + +Many commands need to run a crawl of some kind, running either a user-provided +spider or a special internal one: + +* :command:`bench` +* :command:`check` +* :command:`crawl` +* :command:`fetch` +* :command:`parse` +* :command:`runspider` +* :command:`shell` +* :command:`view` + +They use an internal instance of :class:`scrapy.crawler.AsyncCrawlerProcess` or +:class:`scrapy.crawler.CrawlerProcess` for this. In most cases this detail +shouldn't matter to the user running the command, but when the user :ref:`needs +a non-default Twisted reactor `, it may be important. + +Scrapy decides which of these two classes to use based on the value of the +:setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED` settings. +With :setting:`TWISTED_REACTOR_ENABLED` set to ``False`` it will use +:class:`~scrapy.crawler.AsyncCrawlerProcess`. Otherwise, if the +:setting:`TWISTED_REACTOR` value is the default one +(``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``), +:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used, otherwise +:class:`~scrapy.crawler.CrawlerProcess` will be used. The :ref:`spider settings +` are not taken into account when doing this, as they are +loaded after this decision is made. This may cause an error if the +project-level setting is set to :ref:`the asyncio reactor ` +(:ref:`explicitly ` or :ref:`by using the Scrapy default +`) and :ref:`the setting of the spider being run +` is set to :ref:`a different one `, because +:class:`~scrapy.crawler.AsyncCrawlerProcess` only supports the asyncio reactor. +In this case you should set the :setting:`FORCE_CRAWLER_PROCESS` setting to +``True`` (at the project level or via the command line) so that Scrapy uses +:class:`~scrapy.crawler.CrawlerProcess` which supports all reactors. + Custom project commands ======================= diff --git a/docs/topics/components.rst b/docs/topics/components.rst index d34b3884b..c0df86922 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -9,12 +9,12 @@ A Scrapy component is any class whose objects are built using That includes the classes that you may assign to the following settings: -- :setting:`DNS_RESOLVER` +- :setting:`ADDONS` + +- :setting:`TWISTED_DNS_RESOLVER` - :setting:`DOWNLOAD_HANDLERS` -- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY` - - :setting:`DOWNLOADER_MIDDLEWARES` - :setting:`DUPEFILTER_CLASS` @@ -35,16 +35,90 @@ That includes the classes that you may assign to the following settings: - :setting:`SCHEDULER_PRIORITY_QUEUE` +- :setting:`SCHEDULER_START_DISK_QUEUE` + +- :setting:`SCHEDULER_START_MEMORY_QUEUE` + - :setting:`SPIDER_MIDDLEWARES` Third-party Scrapy components may also let you define additional Scrapy components, usually configurable through :ref:`settings `, to modify their behavior. +.. _from-crawler: + +Initializing from the crawler +============================= + +Any Scrapy component may optionally define the following class method: + +.. classmethod:: from_crawler(cls, crawler: scrapy.crawler.Crawler, *args, **kwargs) + + Return an instance of the component based on *crawler*. + + *args* and *kwargs* are component-specific arguments that some components + receive. However, most components do not get any arguments, and instead + :ref:`use settings `. + + If a component class defines this method, this class method is called to + create any instance of the component. + + The *crawler* object provides access to all Scrapy core components like + :ref:`settings ` and :ref:`signals `, + allowing the component to access them and hook its functionality into + Scrapy. + +.. _component-settings: + +Settings +======== + +Components can be configured through :ref:`settings `. + +Components can read any setting from the +:attr:`~scrapy.crawler.Crawler.settings` attribute of the +:class:`~scrapy.crawler.Crawler` object they can :ref:`get for initialization +`. That includes both built-in and custom settings. + +For example: + +.. code-block:: python + + class MyExtension: + @classmethod + def from_crawler(cls, crawler): + settings = crawler.settings + return cls(settings.getbool("LOG_ENABLED")) + + def __init__(self, log_is_enabled=False): + if log_is_enabled: + print("log is enabled!") + +Components do not need to declare their custom settings programmatically. +However, they should document them, so that users know they exist and how to +use them. + +It is a good practice to prefix custom settings with the name of the component, +to avoid collisions with custom settings of other existing (or future) +components. For example, an extension called ``WarcCaching`` could prefix its +custom settings with ``WARC_CACHING_``. + +Another good practice, mainly for components meant for :ref:`component priority +dictionaries `, is to provide a boolean setting +called ``_ENABLED`` (e.g. ``WARC_CACHING_ENABLED``) to allow toggling +that component on and off without changing the component priority dictionary +setting. You can usually check the value of such a setting during +initialization, and if ``False``, raise +:exc:`~scrapy.exceptions.NotConfigured`. + +When choosing a name for a custom setting, it is also a good idea to have a +look at the names of :ref:`built-in settings `, to try to +maintain consistency with them. + .. _enforce-component-requirements: -Enforcing component requirements -================================ +Enforcing requirements +====================== Sometimes, your components may only be intended to work under certain conditions. For example, they may require a minimum version of Scrapy to work as @@ -58,8 +132,8 @@ In the case of :ref:`downloader middlewares `, :ref:`extensions `, :ref:`item pipelines `, and :ref:`spider middlewares `, you should raise -:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a -parameter to the exception so that it is printed in the logs, for the user to +:exc:`~scrapy.exceptions.NotConfigured`, passing a description of the issue as +a parameter to the exception so that it is printed in the logs, for the user to see. For other components, feel free to raise whatever other exception feels right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy version mismatch, while :exc:`ValueError` may be better if the issue is the diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst index 61aef4bbb..df67bee02 100644 --- a/docs/topics/contracts.rst +++ b/docs/topics/contracts.rst @@ -30,43 +30,15 @@ You can use the following contracts: .. module:: scrapy.contracts.default -.. class:: UrlContract +.. autoclass:: UrlContract - This contract (``@url``) sets the sample URL used when checking other - contract conditions for this spider. This contract is mandatory. All - callbacks lacking this contract are ignored when running the checks:: +.. autoclass:: CallbackKeywordArgumentsContract - @url url +.. autoclass:: MetadataContract -.. class:: CallbackKeywordArgumentsContract +.. autoclass:: ReturnsContract - This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs ` - attribute for the sample request. It must be a valid JSON dictionary. - :: - - @cb_kwargs {"arg1": "value1", "arg2": "value2", ...} - -.. class:: MetadataContract - - This contract (``@meta``) sets the :attr:`meta ` - attribute for the sample request. It must be a valid JSON dictionary. - :: - - @meta {"arg1": "value1", "arg2": "value2", ...} - -.. class:: ReturnsContract - - This contract (``@returns``) sets lower and upper bounds for the items and - requests returned by the spider. The upper bound is optional:: - - @returns item(s)|request(s) [min [max]] - -.. class:: ScrapesContract - - This contract (``@scrapes``) checks that all the items returned by the - callback have the specified fields:: - - @scrapes field_1 field_2 ... +.. autoclass:: ScrapesContract Use the :command:`check` command to run the contract checks. @@ -89,30 +61,16 @@ override three methods: .. module:: scrapy.contracts -.. class:: Contract(method, *args) +.. autoclass:: Contract - :param method: callback function to which the contract is associated - :type method: collections.abc.Callable + .. automethod:: adjust_request_args - :param args: list of arguments passed into the docstring (whitespace - separated) - :type args: list - - .. method:: Contract.adjust_request_args(args) - - This receives a ``dict`` as an argument containing default arguments - for request object. :class:`~scrapy.Request` is used by default, - but this can be changed with the ``request_cls`` attribute. - If multiple contracts in chain have this attribute defined, the last one is used. - - Must return the same or a modified version of it. - - .. method:: Contract.pre_process(response) + .. method:: pre_process(response) This allows hooking in various checks on the response received from the sample request, before it's being passed to the callback. - .. method:: Contract.post_process(output) + .. method:: post_process(output) This allows processing the output of the callback. Iterators are converted to lists before being passed to this hook. diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a65bab3ca..9dcd9d69c 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -4,10 +4,9 @@ Coroutines ========== -.. versionadded:: 2.0 +Scrapy :ref:`supports ` the :ref:`coroutine syntax ` +(i.e. ``async def``). -Scrapy has :ref:`partial support ` for the -:ref:`coroutine syntax `. .. _coroutine-support: @@ -17,15 +16,13 @@ Supported callables The following callables may be defined as coroutines using ``async def``, and hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): +- The :meth:`~scrapy.Spider.start` spider method, which *must* be + defined as an :term:`asynchronous generator`. + + .. versionadded:: 2.13 + - :class:`~scrapy.Request` callbacks. - If you are using any custom or third-party :ref:`spider middleware - `, see :ref:`sync-async-spider-middleware`. - - .. versionchanged:: 2.7 - Output of async callbacks is now processed asynchronously instead of - collecting all of it first. - - The :meth:`process_item` method of :ref:`item pipelines `. @@ -37,19 +34,102 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): methods of :ref:`downloader middlewares `. -- :ref:`Signal handlers that support deferreds `. - - The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` - method of :ref:`spider middlewares `. + method of :ref:`spider middlewares `, which + *must* be defined as an :term:`asynchronous generator` except in + :ref:`universal spider middlewares `. - It must be defined as an :term:`asynchronous generator`. The input - ``result`` parameter is an :term:`asynchronous iterable`. +- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method + of :ref:`spider middlewares `, which *must* be + defined as an :term:`asynchronous generator`. - See also :ref:`sync-async-spider-middleware` and - :ref:`universal-spider-middleware`. + .. versionadded:: 2.13 + +- :ref:`Signal handlers that support deferreds `. + +- Methods of :ref:`download handlers `. + + .. versionadded:: 2.14 + + +.. _coroutine-deferred-apis: + +Using Deferred-based APIs +========================= + +In addition to native coroutine APIs Scrapy has some APIs that return a +:class:`~twisted.internet.defer.Deferred` object or take a user-supplied +function that returns a :class:`~twisted.internet.defer.Deferred` object. These +APIs are also asynchronous but don't yet support native ``async def`` syntax. +In the future we plan to add support for the ``async def`` syntax to these APIs +or replace them with other APIs where changing the existing ones isn't +possible. + +These APIs have a coroutine-based implementation and a Deferred-based one: + +- :class:`scrapy.crawler.Crawler`: + + - :meth:`~scrapy.crawler.Crawler.crawl_async` (coroutine-based) and + :meth:`~scrapy.crawler.Crawler.crawl` (Deferred-based): the former + may be inconvenient to use in Deferred-based code so both are available, + this may change in a future Scrapy version. + +- :class:`scrapy.crawler.AsyncCrawlerRunner` and its subclass + :class:`scrapy.crawler.AsyncCrawlerProcess` (coroutine-based) and + :class:`scrapy.crawler.CrawlerRunner` and its subclass + :class:`scrapy.crawler.CrawlerProcess` (Deferred-based): the former + doesn't support non-default reactors and so the latter should be used + with those. + +The following user-supplied methods can return +:class:`~twisted.internet.defer.Deferred` objects (the methods that can also +return coroutines are listed in :ref:`coroutine-support`): + +- Custom downloader implementations (see :setting:`DOWNLOADER`): + + - ``fetch()`` + +- Custom scheduler implementations (see :setting:`SCHEDULER`): + + - :meth:`~scrapy.core.scheduler.BaseScheduler.open` + + - :meth:`~scrapy.core.scheduler.BaseScheduler.close` + +- Custom dupefilters (see :setting:`DUPEFILTER_CLASS`): + + - ``open()`` + + - ``close()`` + +- Custom feed storages (see :setting:`FEED_STORAGES`): + + - ``store()`` + +- Subclasses of :class:`scrapy.pipelines.media.MediaPipeline`: + + - ``media_to_download()`` + + - ``item_completed()`` + +- Custom storages used by subclasses of + :class:`scrapy.pipelines.files.FilesPipeline`: + + - ``persist_file()`` + + - ``stat_file()`` + +In most cases you can use these APIs in code that otherwise uses coroutines, by +wrapping a :class:`~twisted.internet.defer.Deferred` object into a +:class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for +more information about this. + +For example: a custom scheduler needs to define an ``open()`` method that can +return a :class:`~twisted.internet.defer.Deferred` object. You can write a +method that works with Deferreds and returns one directly, or you can write a +coroutine and convert it into a function that returns a Deferred with +:func:`~scrapy.utils.defer.deferred_f_from_coro_f`. - .. versionadded:: 2.7 General usage ============= @@ -71,7 +151,7 @@ shorter and cleaner: adapter["field"] = data return item - def process_item(self, item, spider): + def process_item(self, item): adapter = ItemAdapter(item) dfd = db.get_some_data(adapter["id"]) dfd.addCallback(self._update_item, item) @@ -85,7 +165,7 @@ becomes: class DbPipeline: - async def process_item(self, item, spider): + async def process_item(self, item): adapter = ItemAdapter(item) adapter["field"] = await db.get_some_data(adapter["id"]) return item @@ -123,13 +203,16 @@ This means you can use many useful Python libraries providing such code: Common use cases for asynchronous code include: -* requesting data from websites, databases and other services (in callbacks, - pipelines and middlewares); +* requesting data from websites, databases and other services (in + :meth:`~scrapy.Spider.start`, callbacks, pipelines and + middlewares); * storing data in databases (in pipelines and middlewares); * delaying the spider initialization until some external event (in the :signal:`spider_opened` handler); -* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download` - (see :ref:`the screenshot pipeline example`). +* calling asynchronous Scrapy methods like + :meth:`ExecutionEngine.download_async() + ` (see :ref:`the + screenshot pipeline example `). .. _aio-libs: https://github.com/aio-libs @@ -145,7 +228,6 @@ within a spider callback: .. code-block:: python from scrapy import Spider, Request - from scrapy.utils.defer import maybe_deferred_to_future class SingleRequestSpider(Spider): @@ -154,8 +236,9 @@ within a spider callback: async def parse(self, response, **kwargs): additional_request = Request("https://example.org/price") - deferred = self.crawler.engine.download(additional_request) - additional_response = await maybe_deferred_to_future(deferred) + additional_response = await self.crawler.engine.download_async( + additional_request + ) yield { "h1": response.css("h1").get(), "price": additional_response.css("#price").get(), @@ -165,9 +248,9 @@ You can also send multiple requests in parallel: .. code-block:: python + import asyncio + from scrapy import Spider, Request - from scrapy.utils.defer import maybe_deferred_to_future - from twisted.internet.defer import DeferredList class MultipleRequestsSpider(Spider): @@ -179,108 +262,13 @@ You can also send multiple requests in parallel: Request("https://example.com/price"), Request("https://example.com/color"), ] - deferreds = [] + tasks = [] for r in additional_requests: - deferred = self.crawler.engine.download(r) - deferreds.append(deferred) - responses = await maybe_deferred_to_future(DeferredList(deferreds)) + task = self.crawler.engine.download_async(r) + tasks.append(task) + responses = await asyncio.gather(*tasks) yield { "h1": response.css("h1::text").get(), - "price": responses[0][1].css(".price::text").get(), - "price2": responses[1][1].css(".color::text").get(), + "price": responses[0].css(".price::text").get(), + "color": responses[1].css(".color::text").get(), } - - -.. _sync-async-spider-middleware: - -Mixing synchronous and asynchronous spider middlewares -====================================================== - -.. versionadded:: 2.7 - -The output of a :class:`~scrapy.Request` callback is passed as the ``result`` -parameter to the -:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method -of the first :ref:`spider middleware ` from the -:ref:`list of active spider middlewares `. -Then the output of that ``process_spider_output`` method is passed to the -``process_spider_output`` method of the next spider middleware, and so on for -every active spider middleware. - -Scrapy supports mixing :ref:`coroutine methods ` and synchronous methods -in this chain of calls. - -However, if any of the ``process_spider_output`` methods is defined as a -synchronous method, and the previous ``Request`` callback or -``process_spider_output`` method is a coroutine, there are some drawbacks to -the asynchronous-to-synchronous conversion that Scrapy does so that the -synchronous ``process_spider_output`` method gets a synchronous iterable as its -``result`` parameter: - -- The whole output of the previous ``Request`` callback or - ``process_spider_output`` method is awaited at this point. - -- If an exception raises while awaiting the output of the previous - ``Request`` callback or ``process_spider_output`` method, none of that - output will be processed. - - This contrasts with the regular behavior, where all items yielded before - an exception raises are processed. - -Asynchronous-to-synchronous conversions are supported for backward -compatibility, but they are deprecated and will stop working in a future -version of Scrapy. - -To avoid asynchronous-to-synchronous conversions, when defining ``Request`` -callbacks as coroutine methods or when using spider middlewares whose -``process_spider_output`` method is an :term:`asynchronous generator`, all -active spider middlewares must either have their ``process_spider_output`` -method defined as an asynchronous generator or :ref:`define a -process_spider_output_async method `. - -.. note:: When using third-party spider middlewares that only define a - synchronous ``process_spider_output`` method, consider - :ref:`making them universal ` through - :ref:`subclassing `. - - -.. _universal-spider-middleware: - -Universal spider middlewares -============================ - -.. versionadded:: 2.7 - -To allow writing a spider middleware that supports asynchronous execution of -its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding -:ref:`asynchronous-to-synchronous conversions `) -while maintaining support for older Scrapy versions, you may define -``process_spider_output`` as a synchronous method and define an -:term:`asynchronous generator` version of that method with an alternative name: -``process_spider_output_async``. - -For example: - -.. code-block:: python - - class UniversalSpiderMiddleware: - def process_spider_output(self, response, result, spider): - for r in result: - # ... do something with r - yield r - - async def process_spider_output_async(self, response, result, spider): - async for r in result: - # ... do something with r - yield r - -.. note:: This is an interim measure to allow, for a time, to write code that - works in Scrapy 2.7 and later without requiring - asynchronous-to-synchronous conversions, and works in earlier Scrapy - versions as well. - - In some future version of Scrapy, however, this feature will be - deprecated and, eventually, in a later version of Scrapy, this - feature will be removed, and all spider middlewares will be expected - to define their ``process_spider_output`` method as an asynchronous - generator. diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst index 89a4d32d8..a5ffe00f1 100644 --- a/docs/topics/developer-tools.rst +++ b/docs/topics/developer-tools.rst @@ -246,7 +246,6 @@ also request each page to get every quote on the site: .. code-block:: python import scrapy - import json class QuoteSpider(scrapy.Spider): @@ -256,7 +255,7 @@ also request each page to get every quote on the site: start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"] def parse(self, response): - data = json.loads(response.text) + data = response.json() for quote in data["quotes"]: yield {"quote": quote["text"]} if data["has_next"]: @@ -280,7 +279,7 @@ In more complex websites, it could be difficult to easily reproduce the requests, as we could need to add ``headers`` or ``cookies`` to make it work. In those cases you can export the requests in `cURL `_ format, by right-clicking on each of them in the network tool and using the -:meth:`~scrapy.Request.from_curl()` method to generate an equivalent +:meth:`~scrapy.Request.from_curl` method to generate an equivalent request: .. code-block:: python @@ -317,4 +316,3 @@ to identifying the correct request and replicating it in your spider. .. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll .. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions - diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst new file mode 100644 index 000000000..34ab4f105 --- /dev/null +++ b/docs/topics/download-handlers.rst @@ -0,0 +1,403 @@ +.. _topics-download-handlers: + +================= +Download handlers +================= + +Download handlers are Scrapy :ref:`components ` used to +download :ref:`requests ` and produce responses from +them. + +Using download handlers +======================= + +The :setting:`DOWNLOAD_HANDLERS_BASE` and :setting:`DOWNLOAD_HANDLERS` settings +tell Scrapy which handler is responsible for a given URL scheme. Their values +are merged into a mapping from scheme names to handler classes. When Scrapy +initializes it creates instances of all configured download handlers (except +for :ref:`lazy ones `) and stores them in a similar +mapping. When Scrapy needs to download a request it extracts the scheme from +its URL, finds the handler for this scheme, passes the request to it and gets a +response from it. If there is no handler for the scheme, the request is not +downloaded and a :exc:`~scrapy.exceptions.NotSupported` exception is raised. + +The :setting:`DOWNLOAD_HANDLERS_BASE` setting contains the default mapping of +handlers. You can use the :setting:`DOWNLOAD_HANDLERS` setting to add handlers +for additional schemes and to replace or disable default ones: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + # disable support for ftp:// requests + "ftp": None, + # replace the default one for http:// + "http": "my.download_handlers.HttpHandler", + # http:// and https:// are different schemes, + # even though they may use the same handler + "https": "my.download_handlers.HttpHandler", + # support for any custom scheme can be added + "sftp": "my.download_handlers.SftpHandler", + } + +.. seealso:: :ref:`security-unencrypted-protocols` and + :ref:`security-local-resources`, for the security implications of the + default ``http``, ``ftp``, ``file`` and ``data`` handlers. + +Replacing HTTP(S) download handlers +----------------------------------- + +While Scrapy provides a default handler for ``http`` and ``https`` schemes, +users may want to use a different handler, provided by Scrapy or by some +3rd-party package. There are several considerations to keep in mind related to +this. + +First of all, as ``http`` and ``https`` are separate schemes, they need +separate entries in the :setting:`DOWNLOAD_HANDLERS` setting, even though it's +likely that the same handler class will be used for both schemes. + +Additionally, some of the Scrapy settings, like :setting:`DOWNLOAD_MAXSIZE`, +are honored by the default HTTP(S) handler but not necessarily by alternative +ones. The same may apply to other Scrapy features, e.g. the +:signal:`bytes_received` and :signal:`headers_received` signals. + +.. _lazy-download-handlers: + +Lazy instantiation of download handlers +--------------------------------------- + +A download handler can be marked as "lazy" by setting its ``lazy`` class +attribute to ``True``. Such handlers are only instantiated when they need to +download their first request. This may be useful when the instantiation is slow +or requires dependencies that are not always available, and the handler is not +needed on every spider run. For example, :class:`the built-in S3 handler +<.S3DownloadHandler>` is lazy. + +Writing your own download handler +================================= + +A download handler is a :ref:`component ` that defines +the following API: + +.. class:: SampleDownloadHandler + + .. attribute:: lazy + :type: bool + + If ``False``, the handler will be instantiated when Scrapy is + initialized. + + If ``True``, the handler will only be instantiated when the first + request handled by it needs to be downloaded. + + .. method:: download_request(request: Request) -> Response + :async: + + Download the given request and return a response. + + .. method:: close() -> None + :async: + + Clean up any resources used by the handler. + +An optional base class for custom handlers is provided: + +.. autoclass:: scrapy.core.downloader.handlers.base.BaseDownloadHandler + :members: + :undoc-members: + :member-order: bysource + +.. _download-handlers-exceptions: + +Exceptions raised by download handlers +====================================== + +.. versionadded:: 2.15.0 + +The built-in download handlers raise Scrapy-specific exceptions instead of +implementation-specific ones, so that code that handles these exceptions can be +written in a generic way. We recommend custom download handlers to also use +these exceptions. + +.. autoexception:: scrapy.exceptions.CannotResolveHostError + +.. autoexception:: scrapy.exceptions.DownloadCancelledError + +.. autoexception:: scrapy.exceptions.DownloadConnectionRefusedError + +.. autoexception:: scrapy.exceptions.DownloadFailedError + +.. autoexception:: scrapy.exceptions.DownloadTimeoutError + +.. autoexception:: scrapy.exceptions.ResponseDataLossError + +.. autoexception:: scrapy.exceptions.UnsupportedURLSchemeError + +.. _download-handlers-ref: + +Built-in HTTP download handlers reference +========================================= + +Scrapy ships several handlers for HTTP and HTTPS requests. While all of them +support basic features, they may differ in support of specific Scrapy features +and settings and HTTP protocol features. See the documentation of specific +handlers and specific settings for more information. Additionally, as the +underlying HTTP client implementations differ between handlers, the behavior of +specific websites may be different when doing the same Scrapy requests but +using different handlers. + +Here is a comparison of some features of the built-in HTTP handlers, see the +individual handler docs for more differences: + +================== ================= ===================== ==================== +Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler +================== ================= ===================== ==================== +Requires asyncio No No Yes +Requires a reactor Yes Yes No +HTTP/1.1 No Yes Yes +HTTP/2 Yes No Yes +TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl`` +HTTP proxies No Yes Yes +SOCKS proxies No No Yes +================== ================= ===================== ==================== + +You can find additional HTTP download handlers in the +scrapy-download-handlers-incubator_ package. This package is made by the Scrapy +developers and contains experimental handlers that may be included in some +later Scrapy version but can already be used. Please refer to the documentation +of this package for more information. + +.. _scrapy-download-handlers-incubator: https://github.com/scrapy-plugins/scrapy-download-handlers-incubator + +.. _twisted-http2-handler: + +H2DownloadHandler +----------------- + +.. note:: Requires the :ref:`twisted-http2 ` extra. + +.. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler + +| Supported scheme: ``https``. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocol +for them. + +It's implemented using :mod:`twisted.web.client` and the ``h2`` library. + +If you want to use this handler you need to replace the default one for the +``https`` scheme: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler", + } + +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. warning:: + + This handler is experimental, and not yet recommended for production + environments. Future Scrapy versions may introduce related changes without + a deprecation period or warning. + +=========================== ================================================ +HTTP proxies No (not implemented) +SOCKS proxies No (not supported by the library) +HTTP/2 Yes +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ + +Other limitations: + +- No support for HTTP/1.1. + +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. + +- No support for the :signal:`bytes_received` and :signal:`headers_received` + signals. + +Known limitations of the HTTP/2 support: + +- No support for HTTP/2 Cleartext (h2c), since no major browser supports + HTTP/2 unencrypted (refer `http2 faq`_). + +- No setting to specify a maximum `frame size`_ larger than the default + value, 16384. Connections to servers that send a larger frame will fail. + +- No support for `server pushes`_, which are ignored. + +.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2 +.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption +.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2 + +HTTP11DownloadHandler +--------------------- + +.. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler + +| Supported schemes: ``http``, ``https``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``http://host/path`` and ``https://host/path`` URLs and +uses the HTTP/1.1 protocol for them. + +It's implemented using :mod:`twisted.web.client`. + +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +=========================== ================================================ +HTTP proxies Yes +SOCKS proxies No (not supported by the library) +HTTP/2 No (implemented as a separate handler) +``response.certificate`` :class:`twisted.internet.ssl.Certificate` object +Per-request ``bindaddress`` Yes +TLS implementation ``pyOpenSSL``/``cryptography`` +=========================== ================================================ + +Other limitations: + +- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER` + to ``scrapy.resolver.CachingHostnameResolver``. + +- HTTPS proxies to HTTPS destinations are not supported. + +.. _httpx-handler: + +HttpxDownloadHandler +-------------------- + +.. note:: Requires the :ref:`httpx ` extra. + +.. versionadded:: 2.15.0 + +.. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler + +| Supported schemes: ``http``, ``https``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: yes. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``http://host/path`` and ``https://host/path`` URLs and +uses the HTTP/1.1 or HTTP/2 protocol for them. + +It's implemented using the httpx2_ library. + +.. _httpx2: https://httpx2.pydantic.dev/ + +If you want to use this handler you need to replace the default ones for the +``http`` and ``https`` schemes: + +.. code-block:: python + + DOWNLOAD_HANDLERS = { + "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler", + } + +Features and limitations +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. warning:: + + This handler is experimental, and not yet recommended for production + environments. Future Scrapy versions may introduce related changes without + a deprecation period or warning or even remove it altogether. + +=========================== ======================================= +HTTP proxies Yes +SOCKS proxies Yes (SOCKS5) +HTTP/2 Yes +``response.certificate`` DER bytes +Per-request ``bindaddress`` No (not supported by the library) +TLS implementation Standard library ``ssl`` +=========================== ======================================= + +Other limitations: + +- The handler creates a separate connection pool for each proxy URL (due to + limitations of ``httpx``) which may lead to higher resource usage when + using proxy rotation. + +.. setting:: HTTPX_HTTP2_ENABLED + +HTTPX_HTTP2_ENABLED +^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 2.17.0 + +Default: ``False`` + +Whether to enable HTTP/2 support in this handler. + +Built-in non-HTTP download handlers reference +============================================= + +DataURIDownloadHandler +---------------------- + +.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler + +| Supported scheme: ``data``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports RFC 2397 ``data:content/type;base64,`` data URIs. + +FileDownloadHandler +------------------- + +.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler + +| Supported scheme: ``file``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``file:///path`` local file URIs. It doesn't +support remote files. + +FTPDownloadHandler +------------------ + +.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler + +| Supported scheme: ``ftp``. +| :ref:`Lazy `: no. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: yes. + +This handler supports ``ftp://host/path`` FTP URIs. + +It's implemented using :mod:`twisted.protocols.ftp`. + +.. _s3-handler: + +S3DownloadHandler +----------------- + +.. note:: Requires the :ref:`s3 ` extra. + +.. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler + +| Supported scheme: ``s3``. +| :ref:`Lazy `: yes. +| :ref:`Requires asyncio support `: no. +| :ref:`Requires a Twisted reactor `: no. + +This handler supports ``s3://bucket/path`` S3 URIs. + +It's implemented using the botocore_ library. + +.. _botocore: https://github.com/boto/botocore diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index edafd9f5f..291855e6d 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -61,20 +61,17 @@ particular setting. See each middleware documentation for more info. Writing your own downloader middleware ====================================== -Each downloader middleware is a Python class that defines one or more of the -methods defined below. - -The main entry point is the ``from_crawler`` class method, which receives a -:class:`~scrapy.crawler.Crawler` instance. The :class:`~scrapy.crawler.Crawler` -object gives you access, for example, to the :ref:`settings `. +Each downloader middleware is a :ref:`component ` that +defines one or more of these methods: .. module:: scrapy.downloadermiddlewares .. class:: DownloaderMiddleware - .. note:: Any of the downloader middleware methods may also return a deferred. + .. note:: Any of the downloader middleware methods may be defined as a + coroutine function (``async def``). - .. method:: process_request(request, spider) + .. method:: process_request(request) This method is called for each request that goes through the download middleware. @@ -106,10 +103,7 @@ object gives you access, for example, to the :ref:`settings `. :param request: the request being processed :type request: :class:`~scrapy.Request` object - :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.Spider` object - - .. method:: process_response(request, response, spider) + .. method:: process_response(request, response) :meth:`process_response` should either: return a :class:`~scrapy.http.Response` object, return a :class:`~scrapy.Request` object or @@ -133,14 +127,12 @@ object gives you access, for example, to the :ref:`settings `. :param response: the response being processed :type response: :class:`~scrapy.http.Response` object - :param spider: the spider for which this response is intended - :type spider: :class:`~scrapy.Spider` object + .. method:: process_exception(request, exception) - .. method:: process_exception(request, exception, spider) - - Scrapy calls :meth:`process_exception` when a download handler - or a :meth:`process_request` (from a downloader middleware) raises an - exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception) + Scrapy calls :meth:`process_exception` when a :ref:`download handler + ` or a :meth:`process_request` (from a + downloader middleware) raises an exception (including an + :exc:`~scrapy.exceptions.IgnoreRequest` exception). :meth:`process_exception` should return: either ``None``, a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object. @@ -164,20 +156,6 @@ object gives you access, for example, to the :ref:`settings `. :param exception: the raised exception :type exception: an ``Exception`` object - :param spider: the spider for which this request is intended - :type spider: :class:`~scrapy.Spider` object - - .. method:: from_crawler(cls, crawler) - - If present, this classmethod is called to create a middleware instance - from a :class:`~scrapy.crawler.Crawler`. It must return a new instance - of the middleware. Crawler object provides access to all Scrapy core - components like settings and signals; it is a way for middleware to - access them and hook its functionality into Scrapy. - - :param crawler: crawler that uses this middleware - :type crawler: :class:`~scrapy.crawler.Crawler` object - .. _topics-downloader-middleware-ref: Built-in downloader middleware reference @@ -313,13 +291,12 @@ DownloadTimeoutMiddleware .. class:: DownloadTimeoutMiddleware This middleware sets the download timeout for requests specified in the - :setting:`DOWNLOAD_TIMEOUT` setting or :attr:`download_timeout` - spider attribute. + :setting:`DOWNLOAD_TIMEOUT` setting. .. note:: - You can also set download timeout per-request using - :reqmeta:`download_timeout` Request.meta key; this is supported + You can also set download timeout per-request using the + :reqmeta:`download_timeout` :attr:`.Request.meta` key; this is supported even when DownloadTimeoutMiddleware is disabled. HttpAuthMiddleware @@ -330,26 +307,15 @@ HttpAuthMiddleware .. class:: HttpAuthMiddleware - This middleware authenticates all requests generated from certain spiders - using `Basic access authentication`_ (aka. HTTP auth). + This middleware authenticates requests using `Basic access authentication`_ + (aka. HTTP auth). - To enable HTTP authentication for a spider, set the ``http_user`` and - ``http_pass`` spider attributes to the authentication data and the - ``http_auth_domain`` spider attribute to the domain which requires this - authentication (its subdomains will be also handled in the same way). - You can set ``http_auth_domain`` to ``None`` to enable the - authentication for all requests but you risk leaking your authentication - credentials to unrelated domains. + Use the :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS`, and + :setting:`HTTPAUTH_DOMAIN` settings to configure it. You can also override + the credentials per request via :attr:`~scrapy.Request.meta` keys + :reqmeta:`http_user`, :reqmeta:`http_pass`, and :reqmeta:`http_auth_domain`. - .. warning:: - In previous Scrapy versions HttpAuthMiddleware sent the authentication - data with all requests, which is a security problem if the spider - makes requests to several different domains. Currently if the - ``http_auth_domain`` attribute is not set, the middleware will use the - domain of the first request, which will work for some spiders but not - for others. In the future the middleware will produce an error instead. - - Example: + Example using settings (e.g. in :attr:`~scrapy.Spider.custom_settings`): .. code-block:: python @@ -357,17 +323,72 @@ HttpAuthMiddleware class SomeIntranetSiteSpider(CrawlSpider): - http_user = "someuser" - http_pass = "somepass" - http_auth_domain = "intranet.example.com" name = "intranet.example.com" + custom_settings = { + "HTTPAUTH_USER": "someuser", + "HTTPAUTH_PASS": "somepass", + "HTTPAUTH_DOMAIN": "intranet.example.com", + } - .. reqmeta:: http_user - .. reqmeta:: http_pass + You can also let + :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserInfoMiddleware` + fill :reqmeta:`http_user` and :reqmeta:`http_pass` from the credentials in + the request URL. - You can alternatively specify ``http_user`` and ``http_pass`` in - :attr:`Request.meta `, or use - :class:`~scrapy.downloadermiddlewares.uriuserinfo.UriUserInfoMiddleware`. + Example using per-request meta: + + .. code-block:: python + + async def start(self): + yield Request( + "https://intranet.example.com/protected/", + meta={ + "http_user": "someuser", + "http_pass": "somepass", + "http_auth_domain": "intranet.example.com", + }, + ) + +.. setting:: HTTPAUTH_USER + +HTTPAUTH_USER +~~~~~~~~~~~~~ + +.. versionadded:: 2.17.0 + +Default: ``""`` + +The username to use for HTTP basic authentication, applied to all requests +whose URL matches :setting:`HTTPAUTH_DOMAIN`. + +.. setting:: HTTPAUTH_PASS + +HTTPAUTH_PASS +~~~~~~~~~~~~~ + +.. versionadded:: 2.17.0 + +Default: ``""`` + +The password to use for HTTP basic authentication. + +.. setting:: HTTPAUTH_DOMAIN + +HTTPAUTH_DOMAIN +~~~~~~~~~~~~~~~ + +.. versionadded:: 2.17.0 + +Default: ``None`` + +The domain (and its subdomains) to which HTTP basic authentication credentials +are sent. Set to ``None`` to send credentials with all requests, but be aware +that this risks leaking credentials to unrelated domains. + +This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER` +or :setting:`HTTPAUTH_PASS` is set. + +.. seealso:: :ref:`security-credential-leakage` .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication @@ -487,7 +508,7 @@ Filesystem storage backend (default) * ``response_body`` - the plain response body - * ``response_headers`` - the request headers (in raw HTTP format) + * ``response_headers`` - the response headers (in raw HTTP format) * ``meta`` - some metadata of this cache resource in Python ``repr()`` format (grep-friendly format) @@ -529,7 +550,7 @@ defines the methods described below. .. method:: open_spider(spider) This method gets called after a spider has been opened for crawling. It handles - the :signal:`open_spider ` signal. + the :signal:`spider_opened` signal. :param spider: the spider which has been opened :type spider: :class:`~scrapy.Spider` object @@ -537,7 +558,7 @@ defines the methods described below. .. method:: close_spider(spider) This method gets called after a spider has been closed. It handles - the :signal:`close_spider ` signal. + the :signal:`spider_closed` signal. :param spider: the spider which has been closed :type spider: :class:`~scrapy.Spider` object @@ -573,8 +594,8 @@ In order to use your storage backend, set: HTTPCache middleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The :class:`HttpCacheMiddleware` can be configured through the following -settings: +:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` can be +configured through the following settings: .. setting:: HTTPCACHE_ENABLED @@ -709,6 +730,8 @@ We assume that the spider will not issue Cache-Control directives in requests unless it actually needs them, so directives in requests are not filtered. +.. _http-compression: + HttpCompressionMiddleware ------------------------- @@ -720,14 +743,12 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. - This middleware also supports decoding `brotli-compressed`_ as well as - `zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is - installed, respectively. + This middleware also supports decoding `brotli-compressed`_ responses with + the :ref:`brotli ` extra, and `zstd-compressed`_ + responses with the :ref:`zstd ` extra. .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt -.. _brotli: https://pypi.org/project/Brotli/ .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt -.. _zstandard: https://pypi.org/project/zstandard/ HttpCompressionMiddleware Settings @@ -754,7 +775,7 @@ HttpProxyMiddleware .. class:: HttpProxyMiddleware This middleware sets the HTTP proxy to use for requests, by setting the - ``proxy`` meta value for :class:`~scrapy.Request` objects. + :reqmeta:`proxy` meta value for :class:`~scrapy.Request` objects. Like the Python standard library module :mod:`urllib.request`, it obeys the following environment variables: @@ -763,11 +784,56 @@ HttpProxyMiddleware * ``https_proxy`` * ``no_proxy`` - You can also set the meta key ``proxy`` per-request, to a value like + You can also set the meta key :reqmeta:`proxy` per-request, to a value like ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``. Keep in mind this value will take precedence over ``http_proxy``/``https_proxy`` environment variables, and it will also ignore ``no_proxy`` environment variable. +.. note:: + + Handling of this meta key needs to be implemented inside the :ref:`download + handler `, so it's not guaranteed to be supported + by all 3rd-party handlers. It's currently unsupported by + :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`. + +.. note:: + + Usually a proxy URL uses the ``http://`` scheme. More rarely, it uses the + ``https://`` one. While both kinds of proxy URLs can be used with both HTTP + and HTTPS destination URLs, the specifics of the network exchange are + different for all 4 cases and it's possible that HTTPS proxies are fully or + partially unsupported by a given download handler. Currently, + :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` + supports HTTPS proxies only for HTTP destinations. + +.. note:: + + If the download handler supports it, you can use a SOCKS proxy URL (e.g. + ``socks5://username:password@some_proxy_server:port``). + :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` + supports SOCKS proxies while other built-in handlers don't. + +HttpProxyMiddleware settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: HTTPPROXY_ENABLED + +HTTPPROXY_ENABLED +^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether or not to enable the :class:`HttpProxyMiddleware`. + +.. setting:: HTTPPROXY_AUTH_ENCODING + +HTTPPROXY_AUTH_ENCODING +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``"latin-1"`` + +The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. + OffsiteMiddleware ----------------- @@ -805,9 +871,9 @@ OffsiteMiddleware .. reqmeta:: allow_offsite If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to - ``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then - the OffsiteMiddleware will allow the request even if its domain is not listed - in allowed domains. + ``True`` or :attr:`Request.meta ` has ``allow_offsite`` + set to ``True``, then the OffsiteMiddleware will allow the request even if + its domain is not listed in allowed domains. RedirectMiddleware ------------------ @@ -922,14 +988,10 @@ Whether the Meta Refresh middleware will be enabled. METAREFRESH_IGNORE_TAGS ^^^^^^^^^^^^^^^^^^^^^^^ -Default: ``[]`` +Default: ``["noscript"]`` Meta tags within these tags are ignored. -.. versionchanged:: 2.0 - The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from - ``["script", "noscript"]`` to ``[]``. - .. versionchanged:: 2.11.2 The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from ``[]`` to ``["noscript"]``. @@ -956,17 +1018,6 @@ RetryMiddleware A middleware to retry failed requests that are potentially caused by temporary problems such as a connection timeout or HTTP 500 error. -Failed pages are collected on the scraping process and rescheduled at the -end, once the spider has finished crawling all regular (non failed) pages. - -The :class:`RetryMiddleware` can be configured through the following -settings (see the settings documentation for more info): - -* :setting:`RETRY_ENABLED` -* :setting:`RETRY_TIMES` -* :setting:`RETRY_HTTP_CODES` -* :setting:`RETRY_EXCEPTIONS` - .. reqmeta:: dont_retry If :attr:`Request.meta ` has ``dont_retry`` key @@ -1025,16 +1076,15 @@ RETRY_EXCEPTIONS Default:: [ - 'twisted.internet.defer.TimeoutError', - 'twisted.internet.error.TimeoutError', - 'twisted.internet.error.DNSLookupError', - 'twisted.internet.error.ConnectionRefusedError', + 'scrapy.exceptions.CannotResolveHostError', + 'scrapy.exceptions.DownloadConnectionRefusedError', + 'scrapy.exceptions.DownloadFailedError', + 'scrapy.exceptions.DownloadTimeoutError', + 'scrapy.exceptions.ResponseDataLossError', 'twisted.internet.error.ConnectionDone', 'twisted.internet.error.ConnectError', 'twisted.internet.error.ConnectionLost', - 'twisted.internet.error.TCPTimedOutError', - 'twisted.web.client.ResponseFailed', - IOError, + OSError, 'scrapy.core.downloader.handlers.http11.TunnelError', ] @@ -1048,6 +1098,23 @@ has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught exception propagation, see :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`. +.. setting:: RETRY_GIVE_UP_LOG_LEVEL + +RETRY_GIVE_UP_LOG_LEVEL +^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 2.17.0 + +Default: ``"ERROR"`` + +:ref:`Logging level ` used for the message logged when a request +exceeds its retries. + +Can be a level name (e.g. ``"WARNING"``) or a number (e.g. ``logging.WARNING`` +or ``30``). + +See also: :reqmeta:`give_up_log_level`, :func:`get_retry_request`. + .. setting:: RETRY_PRIORITY_ADJUST RETRY_PRIORITY_ADJUST @@ -1109,7 +1176,7 @@ Parsers vary in several aspects: * Support for wildcard matching -* Usage of `length based rule `_: +* Usage of `length based rule `_: in particular for ``Allow`` and ``Disallow`` directives, where the most specific rule based on the length of the path trumps the less specific (shorter) rule @@ -1127,7 +1194,7 @@ Based on `Protego `_: * implemented in Python * is compliant with `Google's Robots.txt Specification - `_ + `_ * supports wildcard matching @@ -1147,9 +1214,9 @@ Based on :class:`~urllib.robotparser.RobotFileParser`: * is compliant with `Martijn Koster's 1996 draft specification `_ -* lacks support for wildcard matching +* lacks support for wildcard matching (before Python 3.14.5) -* doesn't use the length based rule +* doesn't use the length based rule (before Python 3.14.5) It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0. @@ -1175,8 +1242,7 @@ Based on `Robotexclusionrulesparser ` extra. * Set :setting:`ROBOTSTXT_PARSER` setting to ``scrapy.robotstxt.RerpRobotParser`` @@ -1230,62 +1296,8 @@ UserAgentMiddleware .. class:: UserAgentMiddleware - Middleware that allows spiders to override the default user agent. - - In order for a spider to override the default user agent, its ``user_agent`` - attribute must be set. - -.. _ajaxcrawl-middleware: - -AjaxCrawlMiddleware -------------------- - -.. module:: scrapy.downloadermiddlewares.ajaxcrawl - -.. class:: AjaxCrawlMiddleware - - Middleware that finds 'AJAX crawlable' page variants based - on meta-fragment html tag. - - .. note:: - - Scrapy finds 'AJAX crawlable' pages for URLs like - ``'http://example.com/!#foo=bar'`` even without this middleware. - AjaxCrawlMiddleware is necessary when URL doesn't contain ``'!#'``. - This is often a case for 'index' or 'main' website pages. - -AjaxCrawlMiddleware Settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. setting:: AJAXCRAWL_ENABLED - -AJAXCRAWL_ENABLED -^^^^^^^^^^^^^^^^^ - -Default: ``False`` - -Whether the AjaxCrawlMiddleware will be enabled. You may want to -enable it for :ref:`broad crawls `. - -HttpProxyMiddleware settings -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. setting:: HTTPPROXY_ENABLED -.. setting:: HTTPPROXY_AUTH_ENCODING - -HTTPPROXY_ENABLED -^^^^^^^^^^^^^^^^^ - -Default: ``True`` - -Whether or not to enable the :class:`HttpProxyMiddleware`. - -HTTPPROXY_AUTH_ENCODING -^^^^^^^^^^^^^^^^^^^^^^^ - -Default: ``"latin-1"`` - -The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. + Middleware that sets the ``User-Agent`` header. + The header value is taken from the :setting:`USER_AGENT` setting. .. _DBM: https://en.wikipedia.org/wiki/Dbm diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst index 801f6d06d..30b6536c7 100644 --- a/docs/topics/dynamic-content.rst +++ b/docs/topics/dynamic-content.rst @@ -14,7 +14,7 @@ from it. If you fail to do that, and you can nonetheless access the desired data through the :ref:`DOM ` from your web browser, see -:ref:`topics-javascript-rendering`. +:ref:`topics-headless-browsing`. .. _topics-finding-data-source: @@ -83,10 +83,10 @@ request with Scrapy. It might be enough to yield a :class:`~scrapy.Request` with the same HTTP method and URL. However, you may also need to reproduce the body, headers and -form parameters (see :class:`~scrapy.FormRequest`) of that request. +form parameters (see :ref:`form`) of that request. As all major browsers allow to export the requests in curl_ format, Scrapy -incorporates the method :meth:`~scrapy.Request.from_curl()` to generate an equivalent +incorporates the method :meth:`~scrapy.Request.from_curl` to generate an equivalent :class:`~scrapy.Request` from a cURL command. To get more information visit :ref:`request from curl ` inside the network tool section. @@ -97,7 +97,7 @@ it `. You can reproduce any request with Scrapy. However, some times reproducing all necessary requests may not seem efficient in developer time. If that is your case, and crawling speed is not a major concern for you, you can alternatively -consider :ref:`JavaScript pre-rendering `. +consider :ref:`using a headless browser `. If you get the expected response `sometimes`, but not always, the issue is probably not your request, but the target server. The target server might be @@ -111,6 +111,8 @@ you may use `curl2scrapy `_. Handling different response formats =================================== +.. skip: start + Once you have a response with the desired data, how you extract the desired data from it depends on the type of response: @@ -131,7 +133,7 @@ data from it depends on the type of response: .. code-block:: python - selector = Selector(data["html"]) + selector = Selector(text=data["html"]) - If the response is JavaScript, or HTML with a `` - """, - ) - self.assertEqual(get_meta_refresh(r1), (5.0, "http://example.org/newpage")) - self.assertEqual(get_meta_refresh(r2), (None, None)) - self.assertEqual(get_meta_refresh(r3), (None, None)) +def test_get_meta_refresh(): + r1 = HtmlResponse( + "http://www.example.com", + body=b""" + + Dummy + blahablsdfsal& + """, + ) + r2 = HtmlResponse( + "http://www.example.com", + body=b""" + + Dummy + blahablsdfsal& + """, + ) + r3 = HtmlResponse( + "http://www.example.com", + body=b""" +