diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
index a9fc3dd68..1f062eef2 100644
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@@ -4,4 +4,4 @@ e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d
# reapplying black to the code with default line length
303f0a70fcf8067adf0a909c2096a5009162383a
# reapplying black again and removing line length on pre-commit black config
-c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962
\ No newline at end of file
+c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 000000000..623d48cfa
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,12 @@
+version: 2
+updates:
+ - package-ecosystem: github-actions
+ directory: "/"
+ schedule:
+ interval: monthly
+ groups:
+ github-actions:
+ patterns:
+ - "*"
+ cooldown:
+ default-days: 7
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 000000000..98a74f8ce
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,31 @@
+
diff --git a/.github/workflows/auto-close-llm-pr.yml b/.github/workflows/auto-close-llm-pr.yml
new file mode 100644
index 000000000..15120b0d9
--- /dev/null
+++ b/.github/workflows/auto-close-llm-pr.yml
@@ -0,0 +1,50 @@
+name: Auto-close LLM PRs
+# The workflow only reads the pull request body through the API, it never
+# checks out or runs pull request code, so pull_request_target is safe here.
+on: # zizmor: ignore[dangerous-triggers]
+ pull_request_target:
+ types: [opened]
+permissions:
+ contents: read
+ pull-requests: write
+jobs:
+ close-llm-pr:
+ name: Close PR if marked as LLM-written
+ runs-on: ubuntu-latest
+ steps:
+ - name: Check PR body and close if LLM-written
+ uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
+ with:
+ github-token: ${{ secrets.GITHUB_TOKEN }}
+ script: |
+ const marker = "This PR was written entirely using an LLM";
+ const { owner, repo } = context.repo;
+ const prNumber = context.payload.pull_request && context.payload.pull_request.number;
+ if (!prNumber) {
+ console.log('No pull request number found in context; exiting.');
+ return;
+ }
+ const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber });
+ const body = pr.body || "";
+ if (body.includes(marker)) {
+ if (pr.state === 'closed') {
+ console.log(`PR #${prNumber} already closed.`);
+ return;
+ }
+ await github.rest.issues.addLabels({
+ owner,
+ repo,
+ issue_number: prNumber,
+ labels: ['spam']
+ });
+ await github.rest.issues.createComment({
+ owner,
+ repo,
+ issue_number: prNumber,
+ body: "Closing this PR because it contains the disclosure: \"This PR was written entirely using an LLM\"."
+ });
+ await github.rest.pulls.update({ owner, repo, pull_number: prNumber, state: 'closed' });
+ console.log(`Closed PR #${prNumber} because marker was found.`);
+ } else {
+ console.log(`Marker not found in PR #${prNumber}; nothing to do.`);
+ }
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 312af3b2e..331fade61 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -1,4 +1,8 @@
name: Checks
+
+permissions:
+ contents: read
+
on:
push:
branches:
@@ -13,42 +17,60 @@ concurrency:
jobs:
checks:
runs-on: ubuntu-latest
+ env:
+ # Make uv use the interpreter that actions/setup-python installed instead
+ # of downloading one of its own.
+ UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
include:
- - python-version: "3.13"
+ - python-version: "3.14"
env:
TOXENV: pylint
- - python-version: "3.9"
+ - python-version: "3.10"
env:
- TOXENV: typing
- - python-version: "3.9"
+ TOXENV: mypy
+ - python-version: "3.10"
env:
- TOXENV: typing-tests
- - python-version: "3.13" # Keep in sync with .readthedocs.yml
+ TOXENV: mypy-tests
+ # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version.
+ - python-version: "3.14"
env:
TOXENV: docs
- python-version: "3.13"
+ env:
+ TOXENV: docs-tests
+ - python-version: "3.14"
env:
TOXENV: twinecheck
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v5
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
+ - name: Set up uv
+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+ with:
+ cache-dependency-glob: |
+ docs/requirements.txt
+ pyproject.toml
+ tox.ini
+
- name: Run check
env: ${{ matrix.env }}
- run: |
- pip install -U tox
- tox
+ run: uvx --with tox-uv tox
pre-commit:
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v4
- - uses: pre-commit/action@v3.0.1
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
new file mode 100644
index 000000000..c50576ec7
--- /dev/null
+++ b/.github/workflows/codspeed.yml
@@ -0,0 +1,59 @@
+---
+name: codspeed
+
+on:
+ push:
+ branches:
+ - master
+ pull_request:
+ paths:
+ - scrapy/**
+ - tests/benchmarks/**
+ - .github/workflows/codspeed.yml
+ - tox.ini
+ workflow_dispatch:
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+ cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+ benchmark:
+ runs-on: ubuntu-latest
+ env:
+ # Make uv use the interpreter that actions/setup-python installed
+ # instead of downloading one of its own.
+ UV_PYTHON_PREFERENCE: only-system
+ permissions:
+ contents: read
+ id-token: write # OIDC authentication with CodSpeed
+
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+
+ - name: Set up Python 3.14
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: '3.14'
+
+ - name: Set up uv
+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+ with:
+ cache-dependency-glob: |
+ pyproject.toml
+ tox.ini
+
+ - name: Install dependencies
+ # tox must stay on PATH for the CodSpeed action to invoke it.
+ run: |
+ uv tool install --with tox-uv tox
+ tox -n -e benchmark
+ - name: Run benchmarks
+ uses: CodSpeedHQ/action@0ca9cbbf4623b599a6c3ed4fc8a922942705d9f1 # v5.0.2
+ with:
+ mode: simulation
+ run: tox -e benchmark
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index d1589f4f7..697647131 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -1,4 +1,8 @@
name: Publish
+
+permissions:
+ contents: read
+
on:
push:
tags:
@@ -9,8 +13,28 @@ concurrency:
cancel-in-progress: true
jobs:
+ build:
+ name: Build distribution
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
+ - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+ with:
+ python-version: "3.14"
+ - run: |
+ python -m pip install --upgrade build
+ python -m build
+ - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+ with:
+ name: python-package-distributions
+ path: dist/
+
publish:
name: Upload release to PyPI
+ needs:
+ - build
runs-on: ubuntu-latest
environment:
name: pypi
@@ -18,12 +42,9 @@ jobs:
permissions:
id-token: write
steps:
- - uses: actions/checkout@v4
- - uses: actions/setup-python@v5
+ - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
- python-version: "3.13"
- - run: |
- python -m pip install --upgrade build
- python -m build
+ name: python-package-distributions
+ path: dist/
- name: Publish to PyPI
- uses: pypa/gh-action-pypi-publish@release/v1
+ uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index d740808cc..9a09aa67d 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -1,4 +1,8 @@
name: macOS
+
+permissions:
+ contents: read
+
on:
push:
branches:
@@ -12,28 +16,62 @@ concurrency:
jobs:
tests:
+ name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})
runs-on: macos-latest
+ env:
+ PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
+ # Make uv use the interpreter that actions/setup-python installed instead
+ # of downloading one of its own.
+ UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
- python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+ include:
+ - python-version: "3.10"
+ env:
+ TOXENV: py
+ - python-version: "3.14"
+ env:
+ TOXENV: py
+ coverage: true
+ - python-version: "3.14"
+ env:
+ TOXENV: no-reactor
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v5
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
+ - name: Set up uv
+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+ with:
+ cache-dependency-glob: |
+ pyproject.toml
+ tox.ini
+
+ - name: Install mitmproxy
+ env:
+ # mitmproxy needs a newer Python than the oldest matrix entries, so let
+ # uv download one where no system interpreter is new enough.
+ UV_PYTHON_PREFERENCE: system
+ run: uv tool install mitmproxy
+
- name: Run tests
- run: |
- pip install -U tox
- tox -e py
+ env: ${{ matrix.env }}
+ run: uvx --with tox-uv tox
- name: Upload coverage report
- uses: codecov/codecov-action@v5
+ if: ${{ matrix.coverage }}
+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
- uses: codecov/test-results-action@v1
+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+ with:
+ report_type: test_results
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 06da46ca1..cd726a2fe 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -1,4 +1,8 @@
name: Ubuntu
+
+permissions:
+ contents: read
+
on:
push:
branches:
@@ -12,14 +16,17 @@ concurrency:
jobs:
tests:
+ name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})
runs-on: ubuntu-latest
+ env:
+ PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
+ # Make uv use the interpreter that actions/setup-python installed instead
+ # of downloading one of its own.
+ UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
include:
- - python-version: "3.9"
- env:
- TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
@@ -32,66 +39,100 @@ jobs:
- python-version: "3.13"
env:
TOXENV: py
- - python-version: "3.13"
+ - python-version: "3.14"
+ env:
+ TOXENV: py
+ coverage: true
+ - python-version: "3.14"
env:
TOXENV: default-reactor
- - python-version: pypy3.10
+ coverage: true
+ - python-version: "3.14"
env:
- TOXENV: pypy3
- - python-version: pypy3.11
- env:
- TOXENV: pypy3
+ TOXENV: no-reactor
+ coverage: true
- # pinned deps
- - python-version: "3.9.21"
+ # min deps
+ - python-version: "3.10.19"
env:
- TOXENV: pinned
- - python-version: "3.9.21"
+ TOXENV: min
+ coverage: true
+ - python-version: "3.10.19"
env:
- TOXENV: default-reactor-pinned
- - python-version: pypy3.10
+ TOXENV: min-default-reactor
+ coverage: true
+ # pinned due to https://github.com/pypy/pypy/issues/5388
+ - python-version: pypy3.11-7.3.20
env:
- TOXENV: pypy3-pinned
- - python-version: "3.9.21"
+ TOXENV: min-pypy3
+ - python-version: "3.10.19"
env:
- TOXENV: extra-deps-pinned
- - python-version: "3.9.21"
+ TOXENV: min-extra-deps
+ coverage: true
+ - python-version: "3.10.19"
env:
- TOXENV: botocore-pinned
+ TOXENV: min-botocore
+ coverage: true
- - python-version: "3.13"
+ - python-version: "3.14"
env:
TOXENV: extra-deps
- - python-version: pypy3.11
+ coverage: true
+ - python-version: "3.14"
+ env:
+ TOXENV: no-reactor-extra-deps
+ coverage: true
+ # pinned due to https://github.com/pypy/pypy/issues/5388
+ - python-version: pypy3.11-7.3.20
env:
TOXENV: pypy3-extra-deps
- - python-version: "3.13"
+ - python-version: "3.14"
env:
TOXENV: botocore
+ coverage: true
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v5
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
- name: Install system libraries
- if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned')
+ if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'min')
run: |
sudo apt-get update
sudo apt-get install libxml2-dev libxslt-dev
+ - name: Set up uv
+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+ with:
+ cache-dependency-glob: |
+ pyproject.toml
+ tox.ini
+
+ - name: Install mitmproxy
+ env:
+ # mitmproxy needs a newer Python than the oldest matrix entries, so let
+ # uv download one where no system interpreter is new enough.
+ UV_PYTHON_PREFERENCE: system
+ # mitmproxy has no PyPy wheels, so run it on CPython regardless of the
+ # interpreter under test.
+ run: uv tool install --python cpython mitmproxy
+
- name: Run tests
env: ${{ matrix.env }}
- run: |
- pip install -U tox
- tox
+ run: uvx --with tox-uv tox
- name: Upload coverage report
- uses: codecov/codecov-action@v5
+ if: ${{ matrix.coverage }}
+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
- uses: codecov/test-results-action@v1
+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+ with:
+ report_type: test_results
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index bbbb704e5..254e9395e 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -1,4 +1,8 @@
name: Windows
+
+permissions:
+ contents: read
+
on:
push:
branches:
@@ -12,59 +16,74 @@ concurrency:
jobs:
tests:
+ name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})
runs-on: windows-latest
+ env:
+ PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
+ # Make uv use the interpreter that actions/setup-python installed instead
+ # of downloading one of its own.
+ UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
include:
- - python-version: "3.9"
- env:
- TOXENV: py
- python-version: "3.10"
env:
TOXENV: py
- - python-version: "3.11"
+ - python-version: "3.14"
env:
TOXENV: py
- - python-version: "3.12"
+ coverage: true
+ - python-version: "3.14"
env:
- TOXENV: py
- - python-version: "3.13"
- env:
- TOXENV: py
- - python-version: "3.13"
- env:
- TOXENV: default-reactor
+ TOXENV: no-reactor
- # pinned deps
- - python-version: "3.9.13"
+ # min deps
+ - python-version: "3.10.11"
env:
- TOXENV: pinned
- - python-version: "3.9.13"
+ TOXENV: min
+ - python-version: "3.10.11"
env:
- TOXENV: extra-deps-pinned
+ TOXENV: min-extra-deps
- - python-version: "3.13"
+ - python-version: "3.14"
env:
TOXENV: extra-deps
steps:
- - uses: actions/checkout@v4
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+ with:
+ persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
- uses: actions/setup-python@v5
+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
+ - name: Set up uv
+ uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+ with:
+ cache-dependency-glob: |
+ pyproject.toml
+ tox.ini
+
+ - name: Install mitmproxy
+ env:
+ # mitmproxy needs a newer Python than the oldest matrix entries, so let
+ # uv download one where no system interpreter is new enough.
+ UV_PYTHON_PREFERENCE: system
+ run: uv tool install mitmproxy
+
- name: Run tests
env: ${{ matrix.env }}
- run: |
- pip install -U tox
- tox
+ run: uvx --with tox-uv tox
- name: Upload coverage report
- uses: codecov/codecov-action@v5
+ if: ${{ matrix.coverage }}
+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
- uses: codecov/test-results-action@v1
+ uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
+ with:
+ report_type: test_results
diff --git a/.gitignore b/.gitignore
index 0a3f0ac1c..5e52ecf1e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,16 +3,18 @@
*.pyc
_trial_temp*
dropin.cache
-docs/build
+docs/_build
*egg-info
-.tox
-venv
-build
-dist
-.idea
+.tox/
+venv/
+.venv/
+build/
+dist/
+.idea/
+.vscode/
htmlcov/
-.coverage
.pytest_cache/
+.coverage
.coverage.*
coverage.*
*.junit.xml
@@ -26,4 +28,4 @@ test-output.*
Thumbs.db
# OSX miscellaneous
-.DS_Store
\ No newline at end of file
+.DS_Store
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0d1a76247..c2cb5056d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,17 +1,37 @@
+exclude: |
+ (?x)(
+ ^docs/_static|
+ ^docs/_tests|
+ ^tests/sample_data
+ )
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.9.3
+ rev: v0.15.20
hooks:
- - id: ruff
+ - id: ruff-check
args: [ --fix ]
- id: ruff-format
- repo: https://github.com/adamchainz/blacken-docs
- rev: 1.19.1
+ rev: 1.20.0
hooks:
- id: blacken-docs
additional_dependencies:
- - black==24.10.0
+ - black==26.5.1
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v5.0.0
+ rev: v6.0.0
hooks:
+ - id: end-of-file-fixer
- id: trailing-whitespace
+- repo: https://github.com/sphinx-contrib/sphinx-lint
+ rev: v1.0.2
+ hooks:
+ - id: sphinx-lint
+- repo: https://github.com/scrapy/sphinx-scrapy
+ rev: 0.8.10
+ hooks:
+ - id: sphinx-scrapy
+- repo: https://github.com/zizmorcore/zizmor-pre-commit
+ rev: v1.28.0
+ hooks:
+ - id: zizmor
+ args: [--no-progress, --fix]
diff --git a/.readthedocs.yml b/.readthedocs.yml
index 23e4cabea..a2773dcf2 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -1,17 +1,10 @@
version: 2
-formats: all
-sphinx:
- configuration: docs/conf.py
- fail_on_warning: true
-
build:
os: ubuntu-24.04
tools:
- # For available versions, see:
- # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
- python: "3.13" # Keep in sync with .github/workflows/checks.yml
-
-python:
- install:
- - requirements: docs/requirements.txt
- - path: .
+ python: "3.14"
+ commands:
+ - pip install tox
+ - tox -e docs
+ - mkdir -p $READTHEDOCS_OUTPUT/html
+ - cp -a docs/_build/all/. $READTHEDOCS_OUTPUT/html/
diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 000000000..24a426d36
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,6 @@
+cff-version: 1.2.0
+message: If you use Scrapy in published research, please cite it as below.
+title: Scrapy
+authors:
+ - name: Scrapy contributors
+url: https://scrapy.org
diff --git a/README.rst b/README.rst
index 536dec7f0..651294add 100644
--- a/README.rst
+++ b/README.rst
@@ -5,7 +5,7 @@
:alt: Scrapy
:width: 480px
-|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki|
+|version| |python_version| |tests| |coverage| |conda| |deepwiki|
.. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg
:target: https://pypi.org/pypi/Scrapy
@@ -15,17 +15,9 @@
:target: https://pypi.org/pypi/Scrapy
:alt: Supported Python Versions
-.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg
- :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
- :alt: Ubuntu
-
-.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
- :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
- :alt: macOS
-
-.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
- :target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
- :alt: Windows
+.. |tests| image:: https://img.shields.io/github/check-runs/scrapy/scrapy/master?label=tests
+ :target: https://github.com/scrapy/scrapy/actions?query=branch%3Amaster
+ :alt: Tests
.. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg
:target: https://codecov.io/github/scrapy/scrapy?branch=master
@@ -40,7 +32,7 @@
:alt: Ask DeepWiki
Scrapy_ is a web scraping framework to extract structured data from websites.
-It is cross-platform, and requires Python 3.9+. It is maintained by Zyte_
+It is cross-platform, and requires Python 3.10+. It is maintained by Zyte_
(formerly Scrapinghub) and `many other contributors`_.
.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors
diff --git a/SECURITY.md b/SECURITY.md
index a5a5c7fb3..c1a5482c6 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -4,8 +4,8 @@
| Version | Supported |
| ------- | ------------------ |
-| 2.13.x | :white_check_mark: |
-| < 2.13.x | :x: |
+| 2.17.x | :white_check_mark: |
+| < 2.17.x | :x: |
## Reporting a Vulnerability
diff --git a/artwork/README.rst b/artwork/README.rst
deleted file mode 100644
index c1880ef6c..000000000
--- a/artwork/README.rst
+++ /dev/null
@@ -1,20 +0,0 @@
-==============
-Scrapy artwork
-==============
-
-This folder contains the Scrapy artwork resources such as logos and fonts.
-
-scrapy-logo.jpg
----------------
-
-The main Scrapy logo, in JPEG format.
-
-qlassik.zip
------------
-
-The font used for the Scrapy logo. Homepage: https://www.dafont.com/qlassik.font
-
-scrapy-blog.logo.xcf
---------------------
-
-The logo used in the Scrapy blog, in Gimp format.
diff --git a/artwork/qlassik.zip b/artwork/qlassik.zip
deleted file mode 100644
index 2885c06ef..000000000
Binary files a/artwork/qlassik.zip and /dev/null differ
diff --git a/artwork/scrapy-blog-logo.xcf b/artwork/scrapy-blog-logo.xcf
deleted file mode 100644
index 320102604..000000000
Binary files a/artwork/scrapy-blog-logo.xcf and /dev/null differ
diff --git a/artwork/scrapy-logo.jpg b/artwork/scrapy-logo.jpg
deleted file mode 100644
index 4315ef8e1..000000000
Binary files a/artwork/scrapy-logo.jpg and /dev/null differ
diff --git a/conftest.py b/conftest.py
index ed7d14166..5a535c168 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,10 +1,21 @@
+from __future__ import annotations
+
+import os
+from importlib.util import find_spec
from pathlib import Path
+from typing import TYPE_CHECKING
import pytest
from twisted.web.http import H2_ENABLED
-from scrapy.utils.reactor import install_reactor
+from scrapy.utils.reactor import set_asyncio_event_loop_policy
+from scrapy.utils.reactorless import install_reactor_import_hook
from tests.keys import generate_keys
+from tests.mockserver.http import MockServer
+from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd
+
+if TYPE_CHECKING:
+ from collections.abc import Generator
def _py_files(folder):
@@ -12,20 +23,15 @@ def _py_files(folder):
collect_ignore = [
- # not a test, but looks like a test
- "scrapy/utils/testproc.py",
- "scrapy/utils/testsite.py",
- "tests/ftpserver.py",
- "tests/mockserver.py",
- "tests/pipelines.py",
- "tests/spiders.py",
- # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess
+ # may need extra deps
+ "docs/_ext",
+ # contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerProcessSubprocess
*_py_files("tests/AsyncCrawlerProcess"),
- # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess
+ # contains scripts to be run by tests/test_crawler_subprocess.py::AsyncCrawlerRunnerSubprocess
*_py_files("tests/AsyncCrawlerRunner"),
- # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
+ # contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerProcessSubprocess
*_py_files("tests/CrawlerProcess"),
- # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
+ # contains scripts to be run by tests/test_crawler_subprocess.py::CrawlerRunnerSubprocess
*_py_files("tests/CrawlerRunner"),
]
@@ -45,88 +51,94 @@ if not H2_ENABLED:
)
)
+if find_spec("httpx2") is None and find_spec("httpx") is None:
+ collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
-@pytest.fixture
-def chdir(tmpdir):
- """Change to pytest-provided temporary directory"""
- tmpdir.chdir()
+if find_spec("pytest_codspeed") is None:
+ collect_ignore.append("tests/benchmarks")
-def pytest_addoption(parser):
+def pytest_addoption(parser, pluginmanager):
+ if pluginmanager.hasplugin("twisted"):
+ return
+ # add the full choice set so that pytest doesn't complain about invalid choices in some cases
parser.addoption(
"--reactor",
- default="asyncio",
- choices=["default", "asyncio"],
+ default="none",
+ choices=["asyncio", "default", "none"],
)
-@pytest.fixture(scope="class")
-def reactor_pytest(request):
- if not request.cls:
- # doctests
- return None
- request.cls.reactor_pytest = request.config.getoption("--reactor")
- return request.cls.reactor_pytest
+@pytest.fixture(scope="session")
+def mockserver() -> Generator[MockServer]:
+ with MockServer() as mockserver:
+ yield mockserver
-@pytest.fixture(autouse=True)
-def only_asyncio(request, reactor_pytest):
- if request.node.get_closest_marker("only_asyncio") and reactor_pytest == "default":
- pytest.skip("This test is only run without --reactor=default")
+@pytest.fixture # function scope because it modifies os.environ
+def proxy_server(
+ request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch
+) -> Generator[str]:
+ kind = request.param
+ proxy = MitmProxy(mode="socks5" if kind == "socks5" else None)
+ url = proxy.start()
+ if kind == "https":
+ url = url.replace("http://", "https://")
+ monkeypatch.setenv("http_proxy", url)
+ monkeypatch.setenv("https_proxy", url)
-
-@pytest.fixture(autouse=True)
-def only_not_asyncio(request, reactor_pytest):
- if (
- request.node.get_closest_marker("only_not_asyncio")
- and reactor_pytest != "default"
- ):
- pytest.skip("This test is only run with --reactor=default")
-
-
-@pytest.fixture(autouse=True)
-def requires_uvloop(request):
- if not request.node.get_closest_marker("requires_uvloop"):
- return
try:
- import uvloop
-
- del uvloop
- except ImportError:
- pytest.skip("uvloop is not installed")
+ yield kind
+ finally:
+ proxy.stop()
-@pytest.fixture(autouse=True)
-def requires_botocore(request):
- if not request.node.get_closest_marker("requires_botocore"):
- return
- try:
- import botocore
-
- del botocore
- except ImportError:
- pytest.skip("botocore is not installed")
-
-
-@pytest.fixture(autouse=True)
-def requires_boto3(request):
- if not request.node.get_closest_marker("requires_boto3"):
- return
- try:
- import boto3
-
- del boto3
- except ImportError:
- pytest.skip("boto3 is not installed")
+@pytest.fixture(scope="session")
+def reactor_pytest(request) -> str:
+ return request.config.getoption("--reactor")
def pytest_configure(config):
- if config.getoption("--reactor") != "default":
- install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
- else:
- # install the reactor explicitly
- from twisted.internet import reactor # noqa: F401
+ if config.getoption("--reactor") == "asyncio":
+ # Needed on Windows to switch from proactor to selector for Twisted reactor compatibility.
+ # If we decide to run tests with both, we will need to add a new option and check it here.
+ set_asyncio_event_loop_policy()
+ elif config.getoption("--reactor") == "none":
+ install_reactor_import_hook()
-# Generate localhost certificate files, needed by some tests
-generate_keys()
+def pytest_runtest_setup(item):
+ # Skip tests based on reactor markers
+ reactor = item.config.getoption("--reactor")
+
+ if item.get_closest_marker("requires_reactor") and reactor == "none":
+ pytest.skip('This test is only run when the --reactor value is not "none"')
+
+ if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}:
+ pytest.skip(
+ 'This test is only run when the --reactor value is "asyncio" (default) or "none"'
+ )
+
+ if item.get_closest_marker("only_not_asyncio") and reactor in {"asyncio", "none"}:
+ pytest.skip(
+ 'This test is only run when the --reactor value is not "asyncio" (default) or "none"'
+ )
+
+ # Skip tests requiring optional dependencies
+ optional_deps = [
+ "uvloop",
+ "botocore",
+ "boto3",
+ ]
+
+ for module in optional_deps:
+ if item.get_closest_marker(f"requires_{module}") and find_spec(module) is None:
+ pytest.skip(f"{module} is not installed")
+
+ if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None:
+ pytest.skip("mitmdump is not available")
+
+
+# Generate localhost certificate files, needed by some tests (but only once if xdist is used)
+if "PYTEST_XDIST_WORKER" not in os.environ:
+ generate_keys()
diff --git a/docs/README.rst b/docs/README.rst
deleted file mode 100644
index 36dd5aea4..000000000
--- a/docs/README.rst
+++ /dev/null
@@ -1,68 +0,0 @@
-:orphan:
-
-======================================
-Scrapy documentation quick start guide
-======================================
-
-This file provides a quick guide on how to compile the Scrapy documentation.
-
-
-Setup the environment
----------------------
-
-To compile the documentation you need Sphinx Python library. To install it
-and all its dependencies run the following command from this dir
-
-::
-
- pip install -r requirements.txt
-
-
-Compile the documentation
--------------------------
-
-To compile the documentation (to classic HTML output) run the following command
-from this dir::
-
- make html
-
-Documentation will be generated (in HTML format) inside the ``build/html`` dir.
-
-
-View the documentation
-----------------------
-
-To view the documentation run the following command::
-
- make htmlview
-
-This command will fire up your default browser and open the main page of your
-(previously generated) HTML documentation.
-
-
-Start over
-----------
-
-To clean up all generated documentation files and start from scratch run::
-
- make clean
-
-Keep in mind that this command won't touch any documentation source files.
-
-
-Recreating documentation on the fly
------------------------------------
-
-There is a way to recreate the doc automatically when you make changes, you
-need to install watchdog (``pip install watchdog``) and then use::
-
- make watch
-
-Alternative method using tox
-----------------------------
-
-To compile the documentation to HTML run the following command::
-
- tox -e docs
-
-Documentation will be generated (in HTML format) inside the ``.tox/docs/tmp/html`` dir.
diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index 4ceb003c7..edb91bfb9 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -29,14 +29,14 @@ def is_setting_index(node: Node) -> bool:
if node.tagname == "index" and node["entries"]: # type: ignore[index,attr-defined]
# index entries for setting directives look like:
# [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
- entry_type, info, refid = node["entries"][0][:3] # type: ignore[index]
+ entry_type, info, _ = node["entries"][0][:3] # type: ignore[index]
return entry_type == "pair" and info.endswith("; setting")
return False
def get_setting_name_and_refid(node: Node) -> tuple[str, str]:
"""Extract setting name from directive index node"""
- entry_type, info, refid = node["entries"][0][:3] # type: ignore[index]
+ _, info, refid = node["entries"][0][:3] # type: ignore[index]
return info.replace("; setting", ""), refid
@@ -77,6 +77,25 @@ def make_setting_element(
return item
+def make_setting_markdown_item(
+ setting_data: SettingData, app: Sphinx, fromdocname: str
+) -> str:
+ uri = app.builder.get_relative_uri(fromdocname, setting_data["docname"])
+ if uri.startswith("#"):
+ target = f"#{setting_data['refid']}"
+ else:
+ target = f"{uri}#{setting_data['refid']}"
+ return f"* [{setting_data['setting_name']}]({target})"
+
+
+def _iter_sorted_settings(env: Any, fromdocname: str) -> list[SettingData]:
+ return [
+ d
+ for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined]
+ if fromdocname != d["docname"]
+ ]
+
+
def replace_settingslist_nodes(
app: Sphinx, doctree: document, fromdocname: str
) -> None:
@@ -87,13 +106,29 @@ def replace_settingslist_nodes(
settings_list.extend(
[
make_setting_element(d, app, fromdocname)
- for d in sorted(env.scrapy_all_settings, key=itemgetter("setting_name")) # type: ignore[attr-defined]
- if fromdocname != d["docname"]
+ for d in _iter_sorted_settings(env, fromdocname)
]
)
node.replace_self(settings_list)
+def visit_settingslist_node_markdown(translator: Any, _node: Node) -> None:
+ builder = translator.builder
+ env = builder.env
+ fromdocname = getattr(builder, "current_doc_name", env.docname)
+ lines = [
+ make_setting_markdown_item(setting_data, builder.app, fromdocname)
+ for setting_data in _iter_sorted_settings(env, fromdocname)
+ ]
+ if lines:
+ translator.add("\n".join(lines), prefix_eol=2, suffix_eol=2)
+ raise nodes.SkipNode
+
+
+def depart_settingslist_node_markdown(_translator: Any, _node: Node) -> None:
+ return None
+
+
def source_role(
name, rawtext, text: str, lineno, inliner, options=None, content=None
) -> tuple[list[Any], list[Any]]:
@@ -126,34 +161,22 @@ def rev_role(
return [node], []
-def setup(app: Sphinx) -> None:
- app.add_crossref_type(
- directivename="setting",
- rolename="setting",
- indextemplate="pair: %s; setting",
- )
- app.add_crossref_type(
- directivename="signal",
- rolename="signal",
- indextemplate="pair: %s; signal",
- )
- app.add_crossref_type(
- directivename="command",
- rolename="command",
- indextemplate="pair: %s; command",
- )
- app.add_crossref_type(
- directivename="reqmeta",
- rolename="reqmeta",
- indextemplate="pair: %s; reqmeta",
- )
+def setup(app: Sphinx) -> dict[str, Any]:
app.add_role("source", source_role)
app.add_role("commit", commit_role)
app.add_role("issue", issue_role)
app.add_role("rev", rev_role)
- app.add_node(SettingslistNode)
+ app.add_node(
+ SettingslistNode,
+ markdown=(visit_settingslist_node_markdown, depart_settingslist_node_markdown),
+ singlemarkdown=(
+ visit_settingslist_node_markdown,
+ depart_settingslist_node_markdown,
+ ),
+ )
app.add_directive("settingslist", SettingsListDirective)
app.connect("doctree-read", collect_scrapy_settings_refs)
app.connect("doctree-resolved", replace_settingslist_nodes)
+ return {"parallel_read_safe": True}
diff --git a/docs/_ext/scrapyfixautodoc.py b/docs/_ext/scrapyfixautodoc.py
index d7a3fb514..e342e92cf 100644
--- a/docs/_ext/scrapyfixautodoc.py
+++ b/docs/_ext/scrapyfixautodoc.py
@@ -3,16 +3,19 @@ Must be included after 'sphinx.ext.autodoc'. Fixes unwanted 'alias of' behavior.
https://github.com/sphinx-doc/sphinx/issues/4422
"""
+from typing import Any
+
# pylint: disable=import-error
from sphinx.application import Sphinx
def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) -> bool:
if not skip:
- # autodocs was generating a text "alias of" for the following members
+ # autodoc was generating the text "alias of" for the following members
return name in {"default_item_class", "default_selector_class"}
return skip
-def setup(app: Sphinx) -> None:
+def setup(app: Sphinx) -> dict[str, Any]:
app.connect("autodoc-skip-member", maybe_skip_member)
+ return {"parallel_read_safe": True}
diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html
index 6ec565e24..29394799b 100644
--- a/docs/_templates/layout.html
+++ b/docs/_templates/layout.html
@@ -1,6 +1,6 @@
{% extends "!layout.html" %}
-{# Overriden to include a link to scrapy.org, not just to the docs root #}
+{# Overridden to include a link to scrapy.org, not just to the docs root #}
{%- block sidebartitle %}
{# the logo helper function was removed in Sphinx 6 and deprecated since Sphinx 4 #}
diff --git a/docs/conf.py b/docs/conf.py
index 1167ce050..1b41adaad 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -3,7 +3,6 @@
# For the full list of built-in configuration values, see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html
-# pylint: disable=import-error
import os
import sys
from collections.abc import Sequence
@@ -27,14 +26,11 @@ author = "Scrapy developers"
# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
extensions = [
- "hoverxref.extension",
"notfound.extension",
"scrapydocs",
- "sphinx.ext.autodoc",
+ "sphinx_scrapy",
"scrapyfixautodoc", # Must be after "sphinx.ext.autodoc"
"sphinx.ext.coverage",
- "sphinx.ext.intersphinx",
- "sphinx.ext.viewcode",
"sphinx_rtd_dark_mode",
]
@@ -70,6 +66,14 @@ html_css_files = [
"custom.css",
]
+html_context = {
+ "display_github": True,
+ "github_user": "scrapy",
+ "github_repo": "scrapy",
+ "github_version": "master",
+ "conf_py_path": "/docs/",
+}
+
# Set canonical URL from the Read the Docs Domain
html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "")
@@ -120,7 +124,7 @@ coverage_ignore_pyobjects = [
# The interface methods of duplicate request filtering classes are already
# covered in the interface documentation part of the DUPEFILTER_CLASS
# setting documentation.
- r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_settings|request_seen|open|close|log)$",
+ r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_crawler|request_seen|open|close|log)$",
# Private exception used by the command-line interface implementation.
r"^scrapy\.exceptions\.UsageError",
# Methods of BaseItemExporter subclasses are only documented in
@@ -137,43 +141,33 @@ coverage_ignore_pyobjects = [
r"^scrapy\.linkextractors\.lxmlhtml\.LxmlParserLinkExtractor",
]
+# -- Options for the autodoc extension ----------------------------------------
+autodoc_member_order = "bysource"
# -- Options for the InterSphinx extension -----------------------------------
# https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#configuration
-intersphinx_mapping = {
- "attrs": ("https://www.attrs.org/en/stable/", None),
- "coverage": ("https://coverage.readthedocs.io/en/latest", None),
- "cryptography": ("https://cryptography.io/en/latest/", None),
- "cssselect": ("https://cssselect.readthedocs.io/en/latest", None),
- "itemloaders": ("https://itemloaders.readthedocs.io/en/latest/", None),
- "parsel": ("https://parsel.readthedocs.io/en/latest/", None),
- "pytest": ("https://docs.pytest.org/en/latest", None),
- "python": ("https://docs.python.org/3", None),
- "sphinx": ("https://www.sphinx-doc.org/en/master", None),
- "tox": ("https://tox.wiki/en/latest/", None),
- "twisted": ("https://docs.twisted.org/en/stable/", None),
- "twistedapi": ("https://docs.twisted.org/en/stable/api/", None),
- "w3lib": ("https://w3lib.readthedocs.io/en/latest", None),
-}
intersphinx_disabled_reftypes: Sequence[str] = []
+# sphinx-scrapy ---------------------------------------------------------------
-# -- Options for sphinx-hoverxref extension ----------------------------------
-# https://sphinx-hoverxref.readthedocs.io/en/latest/configuration.html
-
-hoverxref_auto_ref = True
-hoverxref_role_types = {
- "class": "tooltip",
- "command": "tooltip",
- "confval": "tooltip",
- "hoverxref": "tooltip",
- "mod": "tooltip",
- "ref": "tooltip",
- "reqmeta": "tooltip",
- "setting": "tooltip",
- "signal": "tooltip",
-}
-hoverxref_roles = ["command", "reqmeta", "setting", "signal"]
+scrapy_intersphinx_enable = [
+ "attrs",
+ "coverage",
+ "cryptography",
+ "cssselect",
+ "form2request",
+ "itemloaders",
+ "parsel",
+ "pytest",
+ "pypug",
+ "scrapy-lint",
+ "sphinx",
+ "tox",
+ "twisted",
+ "twistedapi",
+ "w3lib",
+]
+# -- Other options ------------------------------------------------------------
default_dark_mode = False
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 0172887d6..6d2e08fe8 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -251,14 +251,14 @@ Coding style
Please follow these coding conventions when writing code for inclusion in
Scrapy:
-* We use `black `_ for code formatting.
+* We use `Ruff `_ for code formatting.
There is a hook in the pre-commit config
that will automatically format your code before every commit. You can also
- run black manually with ``tox -e pre-commit``.
+ run Ruff manually with ``tox -e pre-commit``.
* Don't put your name in the code you contribute; git provides enough
metadata to identify author of the code.
- See https://docs.github.com/en/get-started/getting-started-with-git/setting-your-username-in-git
+ See https://docs.github.com/en/get-started/git-basics/setting-your-username-in-git
for setup instructions.
.. _scrapy-pre-commit:
@@ -323,9 +323,10 @@ deprecation removals are documented in the :ref:`release notes `.
Tests
=====
-Tests are implemented using the :doc:`Twisted unit-testing framework
-`. Running tests requires
-:doc:`tox `.
+Tests are implemented using pytest_. Running tests requires :doc:`tox
+`.
+
+.. _pytest: https://pytest.org
.. _running-tests:
@@ -371,6 +372,21 @@ To see coverage report install :doc:`coverage `
see output of ``coverage --help`` for more options like html or xml report.
+Some tests need a ``mitmdump`` executable (from mitmproxy_) to test against a
+fully featured proxy server; they are skipped when one cannot be found
+(``mitmproxy`` is intentionally not a test dependency that would be installed
+into test venvs, as that sometimes leads to various dependency conflicts).
+To run these tests, make ``mitmdump`` available in one of these ways:
+
+* install ``mitmproxy`` so that ``mitmdump`` is on your ``PATH``, e.g. with
+ pipx_ (``pipx install mitmproxy``) or uv_ (``uv tool install mitmproxy``);
+
+* have uv_ installed, in which case the tests will run
+ ``uvx --from mitmproxy mitmdump``;
+
+* set the ``MITMDUMP`` environment variable to the path of a ``mitmdump``
+ executable.
+
Writing tests
-------------
@@ -390,8 +406,7 @@ And their unit-tests are in::
.. _issue tracker: https://github.com/scrapy/scrapy/issues
.. _scrapy-users: https://groups.google.com/forum/#!forum/scrapy-users
-.. _Scrapy subreddit: https://reddit.com/r/scrapy
-.. _AUTHORS: https://github.com/scrapy/scrapy/blob/master/AUTHORS
+.. _Scrapy subreddit: https://www.reddit.com/r/scrapy/
.. _tests/: https://github.com/scrapy/scrapy/tree/master/tests
.. _open issues: https://github.com/scrapy/scrapy/issues
.. _PEP 257: https://peps.python.org/pep-0257/
@@ -399,3 +414,6 @@ And their unit-tests are in::
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
.. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
+.. _mitmproxy: https://mitmproxy.org/
+.. _pipx: https://pipx.pypa.io/
+.. _uv: https://docs.astral.sh/uv/
diff --git a/docs/faq.rst b/docs/faq.rst
index 1d09a0e63..80658a5bf 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -82,14 +82,22 @@ to steal from us!
Does Scrapy work with HTTP proxies?
-----------------------------------
-Yes. Support for HTTP proxies is provided (since Scrapy 0.8) through the HTTP
-Proxy downloader middleware. See
+Yes. Support for HTTP proxies is provided through the HTTP Proxy downloader
+middleware. See
:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`.
+Does Scrapy work with SOCKS proxies?
+------------------------------------
+
+Yes, when using
+:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. See
+:class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and the
+handler documentation.
+
How can I scrape an item with attributes in different pages?
------------------------------------------------------------
-See :ref:`topics-request-response-ref-request-callback-arguments`.
+See :ref:`callback-data`.
How can I simulate a user login in my spider?
---------------------------------------------
@@ -128,7 +136,7 @@ middleware with a :ref:`custom downloader middleware
` that requires less memory. For example:
- If your domain names are similar enough, use your own regular expression
- instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into
+ instead of joining the strings in :attr:`~scrapy.Spider.allowed_domains` into
a complex regular expression.
- If you can meet the installation requirements, use pyre2_ instead of
@@ -212,21 +220,15 @@ the :ref:`topics-signals-ref` to know which ones.
What does the response status code 999 mean?
--------------------------------------------
-999 is a custom response status code used by Yahoo sites to throttle requests.
+999 is a custom response status code used by some sites to throttle requests.
Try slowing down the crawling speed by using a download delay of ``2`` (or
-higher) in your spider:
+higher) for the affected domains, with the :setting:`DOWNLOAD_SLOTS` setting:
.. code-block:: python
- from scrapy.spiders import CrawlSpider
-
-
- class MySpider(CrawlSpider):
- name = "myspider"
-
- download_delay = 2
-
- # [ ... rest of the spider code ... ]
+ DOWNLOAD_SLOTS = {
+ "example.com": {"delay": 2},
+ }
Or by setting a global download delay in your project with the
:setting:`DOWNLOAD_DELAY` setting.
@@ -277,7 +279,8 @@ consume a lot of memory.
In order to avoid parsing all the entire feed at once in memory, you can use
the :func:`~scrapy.utils.iterators.xmliter_lxml` and
:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what
-:class:`~scrapy.spiders.XMLFeedSpider` uses.
+:class:`~scrapy.spiders.XMLFeedSpider` and
+:class:`~scrapy.spiders.CSVFeedSpider` use.
.. autofunction:: scrapy.utils.iterators.xmliter_lxml
@@ -323,8 +326,8 @@ section of the site (which varies each time). In that case, the credentials to
log in would be settings, while the url of the section to scrape would be a
spider argument.
-I'm scraping a XML document and my XPath selector doesn't return any items
---------------------------------------------------------------------------
+I'm scraping an XML document and my XPath selector doesn't return any items
+---------------------------------------------------------------------------
You may need to remove namespaces. See :ref:`removing-namespaces`.
@@ -349,18 +352,22 @@ method for this purpose. For example:
class MultiplyItemsMiddleware:
- def process_spider_output(self, response, result, spider):
+ def process_spider_output(self, response, result):
for item_or_request in result:
if isinstance(item_or_request, Request):
+ yield item_or_request
continue
- adapter = ItemAdapter(item)
+ adapter = ItemAdapter(item_or_request)
for _ in range(adapter["multiply_by"]):
- yield deepcopy(item)
+ yield deepcopy(item_or_request)
Does Scrapy support IPv6 addresses?
-----------------------------------
-Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
+Yes, but when using
+:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` or
+:class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` you need to
+set :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
Note that by doing so, you lose the ability to set a specific timeout for DNS requests
(the value of the :setting:`DNS_TIMEOUT` setting is ignored).
@@ -371,8 +378,9 @@ How to deal with ``: filedescriptor out of range in select()
----------------------------------------------------------------------------------------------
This issue `has been reported`_ to appear when running broad crawls in macOS, where the default
-Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a
-different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
+Twisted reactor was :class:`twisted.internet.selectreactor.SelectReactor` at that time.
+If you have switched to this reactor using the :setting:`TWISTED_REACTOR` setting you can switch
+to a different one in the same way.
.. _faq-stop-response-download:
@@ -398,7 +406,6 @@ How can I make a blank request?
from scrapy import Request
-
blank_request = Request("data:,")
In this case, the URL is set to a data URI scheme. Data URLs allow you to include data
@@ -418,4 +425,3 @@ See :issue:`2680`.
.. _has been reported: https://github.com/scrapy/scrapy/issues/2905
.. _Python standard library modules: https://docs.python.org/3/py-modindex.html
.. _Python package: https://pypi.org/
-.. _user agents: https://en.wikipedia.org/wiki/User_agent
diff --git a/docs/index.rst b/docs/index.rst
index 1a9cf636c..688cab81b 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -24,7 +24,7 @@ Having trouble? We'd like to help!
* Ask or search questions in `StackOverflow using the scrapy tag`_.
* Ask or search questions in the `Scrapy subreddit`_.
* Search for questions on the archives of the `scrapy-users mailing list`_.
-* Ask a question in the `#scrapy IRC channel`_,
+* Ask a question in the `#scrapy IRC channel`_.
* Report bugs with Scrapy in our `issue tracker`_.
* Join the Discord community `Scrapy Discord`_.
@@ -91,15 +91,15 @@ Basic concepts
:doc:`topics/selectors`
Extract the data from web pages using XPath.
-:doc:`topics/shell`
- Test your extraction code in an interactive environment.
-
:doc:`topics/items`
Define the data you want to scrape.
:doc:`topics/loaders`
Populate your items with the extracted data.
+:doc:`topics/shell`
+ Test your extraction code in an interactive environment.
+
:doc:`topics/item-pipeline`
Post-process and store your scraped data.
@@ -128,18 +128,14 @@ Built-in services
topics/logging
topics/stats
- topics/email
topics/telnetconsole
:doc:`topics/logging`
- Learn how to use Python's builtin logging on Scrapy.
+ Learn how to use Python's built-in logging on Scrapy.
:doc:`topics/stats`
Collect statistics about your scraping crawler.
-:doc:`topics/email`
- Send email notifications when certain events occur.
-
:doc:`topics/telnetconsole`
Inspect a running crawler using a built-in Python console.
@@ -155,6 +151,7 @@ Solving specific problems
topics/debug
topics/contracts
topics/practices
+ topics/security
topics/broad-crawls
topics/developer-tools
topics/dynamic-content
@@ -179,6 +176,10 @@ Solving specific problems
:doc:`topics/practices`
Get familiar with some Scrapy common practices.
+:doc:`topics/security`
+ Understand the security implications of Scrapy defaults and how to harden
+ them.
+
:doc:`topics/broad-crawls`
Tune Scrapy for crawling a lot domains in parallel.
@@ -229,6 +230,7 @@ Extending Scrapy
topics/signals
topics/scheduler
topics/exporters
+ topics/download-handlers
topics/components
topics/api
@@ -257,6 +259,9 @@ Extending Scrapy
:doc:`topics/exporters`
Quickly export your scraped items to a file (XML, CSV, etc).
+:doc:`topics/download-handlers`
+ Customize how requests are downloaded or add support for new URL schemes.
+
:doc:`topics/components`
Learn the common API and some good practices when building custom Scrapy
components.
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 488a66f36..cba8c15a1 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -9,7 +9,7 @@ Installation guide
Supported Python versions
=========================
-Scrapy requires Python 3.9+, either the CPython implementation (default) or
+Scrapy requires Python 3.10+, either the CPython implementation (default) or
the PyPy implementation (see :ref:`python:implementations`).
.. _intro-install-scrapy:
@@ -89,6 +89,56 @@ just like any other Python package.
(See :ref:`platform-specific guides `
below for non-Python dependencies that you may need to install beforehand).
+.. _extras:
+
+Optional extras
+===============
+
+Scrapy provides optional :ref:`extras `
+that install additional dependencies to enable specific features. To install
+Scrapy with one or more extras, list them in square brackets:
+
+.. code-block:: console
+
+ pip install scrapy[s3,images]
+
+The following extras are available:
+
+.. list-table::
+ :header-rows: 1
+
+ * - Extra
+ - Provides
+ * - ``bpython``
+ - :ref:`bpython shell `
+ * - ``brotli``
+ - :ref:`Brotli response decompression `
+ * - ``gcs``
+ - :ref:`Google Cloud Storage ` for
+ :ref:`feed exports ` and
+ :ref:`media pipelines `
+ * - ``httpx``
+ - :ref:`httpx-handler`, including its HTTP/2 and SOCKS proxy support
+ * - ``images``
+ - :ref:`Images pipeline `
+ * - ``ipython``
+ - :ref:`IPython shell `
+ * - ``ptpython``
+ - :ref:`ptpython shell `
+ * - ``robotparser``
+ - :ref:`Robotexclusionrulesparser robots.txt parsing `
+ * - ``s3``
+ - :ref:`Amazon S3 ` storage for
+ :ref:`feed exports `,
+ :ref:`media pipelines `, and
+ :ref:`S3 downloads `
+ * - ``twisted-http2``
+ - :ref:`twisted-http2-handler`
+ * - ``uvloop``
+ - `uvloop `_ event loop
+ * - ``zstd``
+ - :ref:`Zstandard response decompression `
+
.. _intro-install-platform-notes:
@@ -230,8 +280,8 @@ Installing Scrapy with PyPy on Windows is not tested.
You can check that Scrapy is installed correctly by running ``scrapy bench``.
If this command gives errors such as
``TypeError: ... got 2 unexpected keyword arguments``, this means
-that setuptools was unable to pick up one PyPy-specific dependency.
-To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
+that the ``PyPyDispatcher`` dependency wasn't installed. To fix this issue, run
+``pip install 'PyPyDispatcher>=2.1.0'``.
.. _intro-install-troubleshooting:
@@ -263,7 +313,6 @@ reinstall Twisted with the :code:`tls` extra option::
For details, see `Issue #2473 `_.
.. _Python: https://www.python.org/
-.. _pip: https://pip.pypa.io/en/latest/installing/
.. _lxml: https://lxml.de/index.html
.. _parsel: https://pypi.org/project/parsel/
.. _w3lib: https://pypi.org/project/w3lib/
@@ -273,8 +322,7 @@ For details, see `Issue #2473 `_.
.. _setuptools: https://pypi.org/pypi/setuptools
.. _homebrew: https://brew.sh/
.. _zsh: https://www.zsh.org/
-.. _Anaconda: https://docs.anaconda.com/anaconda/
+.. _Anaconda: https://www.anaconda.com/docs/main
.. _Miniconda: https://docs.conda.io/projects/conda/en/latest/user-guide/install/index.html
-.. _Visual Studio: https://docs.microsoft.com/en-us/visualstudio/install/install-visual-studio
.. _Microsoft C++ Build Tools: https://visualstudio.microsoft.com/visual-cpp-build-tools/
.. _conda-forge: https://conda-forge.org/
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 427f70ed7..59c821175 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -84,16 +84,17 @@ While this enables you to do very fast crawls (sending multiple concurrent
requests at the same time, in a fault-tolerant way) Scrapy also gives you
control over the politeness of the crawl through :ref:`a few settings
`. You can do things like setting a download delay between
-each request, limiting the amount of concurrent requests per domain or per IP, and
+each request, limiting the amount of concurrent requests per domain, and
even :ref:`using an auto-throttling extension ` that tries
to figure these settings out automatically.
.. note::
This is using :ref:`feed exports ` to generate the
- JSON file, you can easily change the export format (XML or CSV, for example) or the
- storage backend (FTP or `Amazon S3`_, for example). You can also write an
- :ref:`item pipeline ` to store the items in a database.
+ JSON Lines file, you can easily change the export format (XML or CSV, for
+ example) or the storage backend (FTP or `Amazon S3`_, for example). You can
+ also write an :ref:`item pipeline ` to store the
+ items in a database.
.. _topics-whatelse:
@@ -151,7 +152,7 @@ The next steps for you are to :ref:`install Scrapy `,
a full-blown Scrapy project and `join the community`_. Thanks for your
interest!
-.. _join the community: https://scrapy.org/community/
+.. _join the community: https://www.scrapy.org/community
.. _web scraping: https://en.wikipedia.org/wiki/Web_scraping
.. _Amazon Associates Web Services: https://affiliate-program.amazon.com/welcome/ecs
.. _Amazon S3: https://aws.amazon.com/s3/
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index c4e04364b..eaf492c95 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -769,7 +769,7 @@ crawlers on top of it.
Also, a common pattern is to build an item with data from more than one page,
using a :ref:`trick to pass additional data to the callbacks
-`.
+`.
Using spider arguments
diff --git a/docs/news.rst b/docs/news.rst
index ef3b549e7..670843e0e 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,1938 @@
Release notes
=============
+Scrapy VERSION (unreleased)
+---------------------------
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- The following runtime usage of zope.interface_ interfaces is removed:
+
+ - :class:`~scrapy.spiderloader.SpiderLoader` and
+ :class:`~scrapy.spiderloader.DummySpiderLoader` are no longer marked
+ as implementing the ``ISpiderLoader`` interface.
+
+ - :func:`~scrapy.spiderloader.get_spider_loader` no longer checks that the
+ configured spider loader implements the ``ISpiderLoader`` interface.
+
+ - :class:`~scrapy.extensions.feedexport.BlockingFeedStorage`,
+ :class:`~scrapy.extensions.feedexport.FileFeedStorage` and
+ :class:`~scrapy.extensions.feedexport.StdoutFeedStorage` are no longer
+ marked as implementing the ``IFeedStorage`` interface.
+
+ (:issue:`6585`, :issue:`7731`)
+
+.. _release-2.17.0:
+
+Scrapy 2.17.0 (2026-07-07)
+--------------------------
+
+Highlights:
+
+- Security bug fixes
+
+- HTTP/2 and SOCKS proxy support for ``HttpxDownloadHandler``
+
+- Improved settings for changing allowed TLS versions
+
+Security bug fixes
+~~~~~~~~~~~~~~~~~~
+
+- ``s3://`` requests now use HTTPS by default, instead of plaintext HTTP.
+
+ Previously, :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler`
+ sent signed S3 requests over plaintext HTTP unless
+ ``request.meta["is_secure"]`` was set to a true value, exposing the request
+ path, the AWS ``Authorization`` header, the ``X-Amz-Security-Token`` header
+ (when using temporary credentials), and the response contents to network
+ attackers, who could also tamper with responses. See the `76g3-c3x4-crvx`_
+ security advisory for details.
+
+ To restore the previous behavior for a given request, set
+ ``request.meta["is_secure"]`` to ``False``.
+
+ .. _76g3-c3x4-crvx: https://github.com/scrapy/scrapy/security/advisories/GHSA-76g3-c3x4-crvx
+
+Deprecations
+~~~~~~~~~~~~
+
+- The ``DOWNLOADER_CLIENT_TLS_METHOD`` setting is deprecated. You should use
+ the :setting:`DOWNLOAD_TLS_MIN_VERSION` and/or
+ :setting:`DOWNLOAD_TLS_MAX_VERSION` settings instead if you want to change
+ the TLS method selection.
+ (:issue:`3288`, :issue:`6546`)
+
+- The following spider attributes are deprecated in favor of settings:
+
+ - ``http_user`` (use :setting:`HTTPAUTH_USER`)
+
+ - ``http_pass`` (use :setting:`HTTPAUTH_PASS`)
+
+ - ``http_auth_domain`` (use :setting:`HTTPAUTH_DOMAIN`)
+
+ (:issue:`7590`)
+
+- The ``scrapy.commands.ScrapyCommand.help()`` method is deprecated. It was
+ never called by Scrapy.
+ (:issue:`7626`, :issue:`7633`)
+
+- The following TLS-related functions and constants, intended for internal
+ use, are deprecated:
+
+ - ``scrapy.core.downloader.tls.METHOD_TLS``
+
+ - ``scrapy.core.downloader.tls.METHOD_TLSv10``
+
+ - ``scrapy.core.downloader.tls.METHOD_TLSv11``
+
+ - ``scrapy.core.downloader.tls.METHOD_TLSv12``
+
+ - ``scrapy.core.downloader.tls.openssl_methods``
+
+ - ``scrapy.core.downloader.tls.DEFAULT_CIPHERS``
+
+ - ``scrapy.utils.ssl.ffi_buf_to_string()``
+
+ - ``scrapy.utils.ssl.get_temp_key_info()``
+
+ - ``scrapy.utils.ssl.x509name_to_string()``
+
+ (:issue:`6546`, :issue:`7619`, :issue:`7665`)
+
+- The ``CRAWLSPIDER_FOLLOW_LINKS`` setting is deprecated. You can set
+ ``follow=False`` in your rules to achieve the same effect.
+ (:issue:`7592`)
+
+- Instantiating
+ :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
+ without a ``crawler`` argument is deprecated.
+ (:issue:`7655`)
+
+- Instantiating
+ :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` without a
+ ``settings`` argument is deprecated.
+ (:issue:`7664`)
+
+New features
+~~~~~~~~~~~~
+
+- Added support for HTTP/2 requests to
+ :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. It
+ requires setting the new :setting:`HTTPX_HTTP2_ENABLED` setting to
+ ``True``.
+ (:issue:`7575`)
+
+- Added support for SOCKS proxies to
+ :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
+ (:issue:`747`, :issue:`7575`)
+
+- Added :setting:`DOWNLOAD_TLS_MIN_VERSION` and
+ :setting:`DOWNLOAD_TLS_MAX_VERSION` settings as replacements for the
+ ``DOWNLOADER_CLIENT_TLS_METHOD`` setting (which is now deprecated).
+ Compared to the old setting, they support specifying a range of allowed
+ versions and support newer TLS versions.
+ (:issue:`4821`, :issue:`6546`)
+
+- Added :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS` and
+ :setting:`HTTPAUTH_DOMAIN` settings and :reqmeta:`http_user`,
+ :reqmeta:`http_pass` and :reqmeta:`http_auth_domain` meta keys as more
+ flexible ways to set HTTP authentication data.
+ (:issue:`7590`)
+
+- Added a :reqmeta:`verbatim_url` meta key that can be set to ``True`` to
+ skip request URL canonicalization.
+ (:issue:`7473`)
+
+- Added ``deny_tags`` and ``deny_attrs`` arguments to :class:`LinkExtractor
+ `.
+ (:issue:`6321`, :issue:`7679`)
+
+- :attr:`scrapy.Item.fields` now returns the fields in the definition order
+ instead of the alphabetical one.
+ (:issue:`7015`, :issue:`7694`)
+
+- Added a :setting:`RETRY_GIVE_UP_LOG_LEVEL` setting, a
+ :reqmeta:`give_up_log_level` meta key and a ``give_up_log_level`` argument
+ of the
+ :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function that
+ allow changing the log level of the message logged when the retry limit has
+ been reached.
+ (:issue:`4622`, :issue:`5297`, :issue:`7567`)
+
+- It's now possible to set :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` to
+ ``None`` to use the default ciphers of the underlying TLS implementation.
+ (:issue:`7499`, :issue:`7665`)
+
+Improvements
+~~~~~~~~~~~~
+
+- :class:`~scrapy.FormRequest` is no longer deprecated, only its
+ ``from_response()`` method is still deprecated.
+ (:issue:`7561`, :issue:`7671`)
+
+- Switched the item definition in the default project template from a
+ :class:`scrapy.item.Item` to a dataclass.
+ (:issue:`7493`, :issue:`7513`)
+
+- Fixed deprecation warnings with pyOpenSSL 26.3.0.
+ (:issue:`7619`)
+
+- Removed the runtime warnings for :attr:`Spider.allowed_domains
+ ` containing URLs or domains with ports
+ instead of just domains and for spider classes having a ``start_url``
+ attribute instead of :class:`~scrapy.spiders.Spider.start_urls`. Please use
+ :doc:`scrapy-lint ` to find mistakes in your spider code
+ instead.
+ (:issue:`4421`, :issue:`7627`)
+
+- :func:`scrapy.utils.test.get_crawler` now disables
+ :setting:`TELNETCONSOLE_ENABLED` by default.
+ (:issue:`7644`)
+
+- Other code refactoring and improvements.
+ (:issue:`7409`, :issue:`7593`, :issue:`7594`, :issue:`7611`, :issue:`7649`)
+
+Bug fixes
+~~~~~~~~~
+
+- :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` no
+ longer ignores proxy credentials for redirected or retried requests.
+ (:issue:`7601`, :issue:`7630`)
+
+- :class:`~scrapy.extensions.feedexport.GCSFeedStorage` now closes the
+ temporary file after the upload.
+ (:issue:`7546`)
+
+- Fixed ``scrapy shell `` running a full spider crawl when there is a
+ spider for the requested URL. This bug was introduced in Scrapy 2.13.0.
+ (:issue:`7552`, :issue:`7557`)
+
+- The :setting:`IMAGES_STORE_S3_ACL` and :setting:`IMAGES_STORE_GCS_ACL`
+ settings are no longer ignored. This bug was introduced in Scrapy 2.12.0.
+ (:issue:`7597`, :issue:`7614`)
+
+- :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` now closes
+ the connection after making the request.
+ (:issue:`7602`, :issue:`7667`)
+
+- Removed the deprecated ``spider`` argument from the pipeline defined in the
+ default project template.
+ (:issue:`7676`)
+
+- Fixed ``scrapy genspider --edit`` not working.
+ (:issue:`7260`, :issue:`7683`)
+
+- When a :class:`~scrapy.crawler.Crawler` instance is passed to
+ :meth:`AsyncCrawlerRunner.create_crawler()
+ ` or
+ :meth:`CrawlerRunner.create_crawler()
+ `, settings from both classes
+ are now merged, previously only the settings from the
+ :class:`~scrapy.crawler.Crawler` instance were used.
+ (:issue:`1280`, :issue:`7647`)
+
+- Fixed several issues with cookie handling in
+ :func:`scrapy.utils.request.request_to_curl`.
+ (:issue:`7603`, :issue:`7675`, :issue:`7684`)
+
+- Fixed :class:`scrapy.resolver.CachingThreadedResolver` not disabling the
+ cache when :setting:`DNSCACHE_ENABLED` is set to ``False``.
+ (:issue:`7663`)
+
+- Fixed :func:`scrapy.utils.response.open_in_browser` not removing comments
+ when looking for the ```` tag.
+ (:issue:`7506`)
+
+- Fixed checking for deprecated methods in custom :setting:`ITEM_PROCESSOR`
+ implementations.
+ (:issue:`7589`)
+
+- Fixed :func:`scrapy.utils.url.strip_url` corrupting some URLs with
+ credentials.
+ (:issue:`7604`, :issue:`7605`)
+
+- :func:`scrapy.utils.misc.rel_has_nofollow` now ignores the case when
+ looking for "nofollow" strings.
+ (:issue:`7632`)
+
+- Fixed an exception in :class:`scrapy.utils.sitemap.Sitemap` when parsing
+ some malformed sitemaps.
+ (:issue:`7686`, :issue:`7687`)
+
+Documentation
+~~~~~~~~~~~~~
+
+- Mentioned :doc:`scrapy-lint ` in the docs.
+ (:issue:`4421`, :issue:`7627`)
+
+- Added the docs about :ref:`security considerations `.
+ (:issue:`7389`, :issue:`7678`)
+
+- Improved the :ref:`item pipeline docs `.
+ (:issue:`2350`, :issue:`7676`)
+
+- Documented which stats are collected by
+ :class:`~scrapy.extensions.corestats.CoreStats`.
+ (:issue:`7421`)
+
+- Switched documentation examples from using :class:`scrapy.item.Item` to
+ using dataclasses.
+ (:issue:`7493`, :issue:`7513`)
+
+- Added feature comparison tables to the :ref:`download handler
+ ` docs.
+ (:issue:`7575`)
+
+- Improved the docs for :ref:`logging settings `.
+ (:issue:`6909`, :issue:`7668`)
+
+- Documented a way to :ref:`improve startup time and memory usage
+ ` by using :setting:`SPIDER_MODULES`.
+ (:issue:`7576`, :issue:`7600`)
+
+- Clarified handling of the ``type`` argument of :class:`~scrapy.Selector`.
+ (:issue:`7704`)
+
+- Other documentation improvements and fixes.
+ (:issue:`4954`,
+ :issue:`6120`,
+ :issue:`7286`,
+ :issue:`7564`,
+ :issue:`7573`,
+ :issue:`7598`,
+ :issue:`7599`,
+ :issue:`7698`)
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+- Fixed deprecation warnings with pytest 9.1.0.
+ (:issue:`7621`)
+
+- Type hints improvements and fixes.
+ (:issue:`6958`, :issue:`7586`)
+
+- CI and test improvements and fixes.
+ (:issue:`5954`,
+ :issue:`7002`,
+ :issue:`7017`,
+ :issue:`7247`,
+ :issue:`7508`,
+ :issue:`7545`,
+ :issue:`7566`,
+ :issue:`7574`,
+ :issue:`7585`,
+ :issue:`7595`,
+ :issue:`7608`,
+ :issue:`7610`,
+ :issue:`7612`,
+ :issue:`7616`,
+ :issue:`7625`,
+ :issue:`7637`,
+ :issue:`7639`,
+ :issue:`7640`,
+ :issue:`7641`,
+ :issue:`7642`,
+ :issue:`7643`,
+ :issue:`7644`,
+ :issue:`7645`,
+ :issue:`7646`,
+ :issue:`7654`,
+ :issue:`7655`,
+ :issue:`7664`,
+ :issue:`7672`,
+ :issue:`7677`,
+ :issue:`7680`,
+ :issue:`7682`,
+ :issue:`7692`)
+
+.. _release-2.16.0:
+
+Scrapy 2.16.0 (2026-05-19)
+--------------------------
+
+Highlights:
+
+- Official support for Python 3.14
+
+- Support for Twisted 26.4.0+
+
+Modified requirements
+~~~~~~~~~~~~~~~~~~~~~
+
+- Increased the minimum versions of the following dependencies:
+
+ - service_identity_: 18.1.0 → 23.1.0
+
+ (:issue:`7347`)
+
+- Added support for Twisted 26.4.0+.
+ (:issue:`7347`, :issue:`7505`, :issue:`7520`)
+
+- Added support for Python 3.14.
+ (:issue:`6604`, :issue:`7460`)
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- The following classes and functions, intended for internal use by
+ :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
+ and :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`, have
+ been made private:
+
+ - ``scrapy.core.downloader.handlers.http11.ScrapyAgent``
+
+ - ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent``
+
+ - ``scrapy.core.downloader.handlers.http11.TunnelingAgent``
+
+ - ``scrapy.core.downloader.handlers.http11.TunnelingTCP4ClientEndpoint``
+
+ - ``scrapy.core.downloader.handlers.http11.tunnel_request_data()``
+
+ - ``scrapy.core.downloader.handlers.http2.ScrapyH2Agent``
+
+ (:issue:`7496`, :issue:`7510`)
+
+Deprecations
+~~~~~~~~~~~~
+
+- ``scrapy.FormRequest`` is deprecated. You can use the :doc:`form2request
+ ` library instead, see :ref:`form`.
+ (:issue:`6438`)
+
+- ``scrapy.utils.python.MutableChain`` is deprecated.
+ (:issue:`7504`)
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+- The ``start_requests()`` method of :class:`~scrapy.Spider`, deprecated in
+ 2.13.0, is removed and no longer called. Use :meth:`~scrapy.Spider.start`
+ instead, or both to maintain support for lower Scrapy versions.
+ (:issue:`7490`)
+
+- Support for ``process_start_requests()`` methods of :ref:`spider middlewares
+ `, deprecated in 2.13.0, is removed. Use
+ :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead,
+ or both to maintain support for lower Scrapy versions.
+ (:issue:`7490`)
+
+- Support for synchronous ``process_spider_output()`` methods of spider
+ middlewares, deprecated in Scrapy 2.13.0, is removed. You should upgrade
+ the affected middlewares to have asynchronous ``process_spider_output()``
+ methods.
+ (:issue:`7504`)
+
+- The ``spider`` arguments of the following methods of
+ :class:`~scrapy.core.scraper.Scraper`, deprecated in Scrapy 2.13.0, are
+ removed:
+
+ - ``close_spider()``
+
+ - ``enqueue_scrape()``
+
+ - ``handle_spider_error()``
+
+ - ``handle_spider_output()``
+
+ (:issue:`7487`)
+
+- HTTP/1.0 support code, deprecated in Scrapy 2.13.0, is removed. This
+ includes:
+
+ - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler``
+
+ - The ``scrapy.core.downloader.webclient`` module.
+
+ - The ``DOWNLOADER_HTTPCLIENTFACTORY`` setting.
+
+ (:issue:`7486`)
+
+- The following functions, deprecated in Scrapy 2.13.0, are removed, you
+ should import them from :mod:`w3lib.url` directly instead:
+
+ - ``scrapy.utils.url.add_or_replace_parameter()``
+
+ - ``scrapy.utils.url.add_or_replace_parameters()``
+
+ - ``scrapy.utils.url.any_to_uri()``
+
+ - ``scrapy.utils.url.canonicalize_url()``
+
+ - ``scrapy.utils.url.file_uri_to_path()``
+
+ - ``scrapy.utils.url.is_url()``
+
+ - ``scrapy.utils.url.parse_data_uri()``
+
+ - ``scrapy.utils.url.parse_url()``
+
+ - ``scrapy.utils.url.path_to_file_uri()``
+
+ - ``scrapy.utils.url.safe_download_url()``
+
+ - ``scrapy.utils.url.safe_url_string()``
+
+ - ``scrapy.utils.url.url_query_cleaner()``
+
+ - ``scrapy.utils.url.url_query_parameter()``
+
+ (:issue:`7487`)
+
+- The following test-related code, deprecated in Scrapy 2.13.0, is removed:
+
+ - the ``scrapy.utils.testproc`` module
+
+ - the ``scrapy.utils.testsite`` module
+
+ - ``scrapy.utils.test.assert_gcs_environ()``
+
+ - ``scrapy.utils.test.get_ftp_content_and_delete()``
+
+ - ``scrapy.utils.test.get_gcs_content_and_delete()``
+
+ - ``scrapy.utils.test.mock_google_cloud_storage()``
+
+ - ``scrapy.utils.test.skip_if_no_boto()``
+
+ - ``scrapy.utils.test.TestSpider``
+
+ (:issue:`7487`)
+
+- ``scrapy.utils.versions.scrapy_components_versions()``, deprecated in
+ Scrapy 2.13.0, is removed, you can use
+ :func:`scrapy.utils.versions.get_versions` instead.
+ (:issue:`7487`)
+
+- ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` and
+ ``scrapy.utils.url.escape_ajax()``, deprecated in Scrapy 2.13.0, are
+ removed.
+ (:issue:`7487`)
+
+- The ``__init__()`` method of priority queue classes (see
+ :setting:`SCHEDULER_PRIORITY_QUEUE`) now needs to support a keyword-only
+ ``start_queue_cls`` parameter, not supporting it was deprecated in Scrapy
+ 2.13.0.
+ (:issue:`7487`)
+
+- ``scrapy.spiders.init.InitSpider``, deprecated in Scrapy 2.13.0, is
+ removed.
+ (:issue:`7487`)
+
+New features
+~~~~~~~~~~~~
+
+- New features and improvements for
+ :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`:
+
+ - Support for proxies.
+
+ - Support for the :reqmeta:`download_latency` meta key.
+
+ - Support for :attr:`Response.certificate
+ `.
+
+ - Default headers set by the ``httpx`` library are no longer added to
+ requests.
+
+ (:issue:`7441`, :issue:`7524`)
+
+- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now
+ skips HTTPS proxy certificate verification when the
+ :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting is set to ``False``.
+ (:issue:`7496`)
+
+Improvements
+~~~~~~~~~~~~
+
+- :func:`time.monotonic` is used instead of :func:`time.time` to calculate
+ elapsed time in various places.
+ (:issue:`7377`)
+
+- Improved extraction of the file extension from the URL in
+ :class:`~scrapy.pipelines.files.FilesPipeline`.
+ (:issue:`4225`, :issue:`7414`)
+
+- Other code refactoring and improvements.
+ (:issue:`7401`)
+
+Bug fixes
+~~~~~~~~~
+
+- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now
+ raises an exception when a request has an ``https://`` destination and an
+ ``https://`` proxy, which is not supported by this handler. Previously it
+ tried to connect to the proxy via HTTP in this case.
+ (:issue:`7496`)
+
+- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now
+ raises an exception for requests with ``http://`` URLs instead of trying to
+ connect, which is not supported by this handler.
+ (:issue:`7496`)
+
+- :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` no longer
+ adds the ``:status`` pseudo-header to :attr:`Response.headers
+ `.
+ (:issue:`7441`)
+
+- Fixed :func:`scrapy.utils.response.open_in_browser` removing the ````
+ tag when adding the ```` tag.
+ (:issue:`7459`)
+
+Documentation
+~~~~~~~~~~~~~
+
+- Documented that
+ :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
+ doesn't support HTTPS proxies for HTTPS destinations and that
+ :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` doesn't
+ support proxies at all.
+ (:issue:`7496`)
+
+- Added an example of using
+ :class:`logging.handlers.TimedRotatingFileHandler` to rotate Scrapy logs.
+ (:issue:`3628`, :issue:`7501`)
+
+- Added a ``CITATION.cff`` file.
+ (:issue:`7502`, :issue:`7519`)
+
+- Mentioned ``DOWNLOADER_CLIENT_TLS_METHOD`` in :ref:`bans`.
+ (:issue:`5232`, :issue:`7518`)
+
+- Other documentation improvements and fixes.
+ (:issue:`7417`,
+ :issue:`7463`,
+ :issue:`7472`,
+ :issue:`7480`,
+ :issue:`7489`,
+ :issue:`7503`,
+ :issue:`7507`)
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+- Added tests that connect to https://books.toscrape.com/ to test the
+ behavior with a real website. These tests are marked with the
+ ``requires_internet`` pytest mark and can be skipped with e.g.
+ ``-m 'not requires_internet'`` if you cannot or don't want to run them.
+ (:issue:`7520`)
+
+- Type hints improvements and fixes.
+ (:issue:`7492`, :issue:`7532`)
+
+- CI and test improvements and fixes.
+ (:issue:`7441`, :issue:`7466`, :issue:`7491`, :issue:`7496`)
+
+.. _release-2.15.2:
+
+Scrapy 2.15.2 (2026-04-28)
+--------------------------
+
+Bug fixes
+~~~~~~~~~
+
+- Fixed links in https://docs.scrapy.org/llms.txt (:issue:`7467`)
+
+.. _release-2.15.1:
+
+Scrapy 2.15.1 (2026-04-23)
+--------------------------
+
+Bug fixes
+~~~~~~~~~
+
+- Sharing of the SSL context between multiple connections, introduced in
+ Scrapy 2.15.0, is reverted as it caused problems and wasn't actually
+ needed.
+ (:issue:`7445`, :issue:`7450`)
+
+- Fixed :meth:`scrapy.settings.BaseSettings.getwithbase` failing on keys with
+ dots that aren't import names. It now works the way it worked before Scrapy
+ 2.15.0, without trying to match class objects and import path. A separate
+ method,
+ :func:`~scrapy.settings.BaseSettings.get_component_priority_dict_with_base`,
+ was added that does that, and it is now used for :ref:`component priority
+ dictionaries `.
+ (:issue:`7426`, :issue:`7449`)
+
+- Documentation rendering improvements.
+ (:issue:`7452`, :issue:`7454`)
+
+.. _release-2.15.0:
+
+Scrapy 2.15.0 (2026-04-09)
+--------------------------
+
+Highlights:
+
+- Experimental support for running without a Twisted reactor
+
+- Experimental ``httpx``-based download handler
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- The built-in HTTP :ref:`download handlers ` now
+ raise Scrapy-specific exceptions instead of implementation-specific ones,
+ see :ref:`download-handlers-exceptions`. This can affect user code that
+ handles downloader exceptions, such as ``process_exception()`` methods of
+ custom :ref:`downloader middlewares `.
+ (:issue:`7208`)
+
+- In order to fix a long-standing bug with handling of asynchronous storages,
+ the following changes were made to media pipeline classes, which can impact
+ some of the user code that subclasses them or calls their methods directly:
+
+ - overrides of :meth:`scrapy.pipelines.media.MediaPipeline.media_downloaded`
+ and :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` can now
+ return coroutines
+
+ - :meth:`~scrapy.pipelines.files.FilesPipeline.media_downloaded`,
+ :meth:`~scrapy.pipelines.files.FilesPipeline.file_downloaded` and
+ :meth:`~scrapy.pipelines.images.ImagesPipeline.image_downloaded` now
+ return coroutines
+
+ (:issue:`2183`, :issue:`6369`, :issue:`7182`)
+
+- ``Request`` and ``Response`` objects: ``__slots__`` and setter changes:
+
+ - :class:`scrapy.http.Request` and :class:`scrapy.http.Response` now
+ define ``__slots__``. Assigning arbitrary attributes to instances (for
+ example, ``response.foo = 1``) will raise ``AttributeError``. Store
+ per-request/response data in the request/response ``meta`` mapping
+ instead of attaching new attributes to the objects.
+
+ - If you maintain custom ``Request`` or ``Response`` subclasses that
+ relied on dynamic instance attributes, either add ``'__dict__'`` to
+ your subclass ``__slots__`` to allow dynamic attributes, or migrate
+ per-instance state to ``meta`` or explicit documented attributes.
+
+ - The setters for ``headers``, ``flags`` and ``cookies`` no longer coerce
+ falsy values into ``None``. For example, ``request.headers = {}`` now
+ stores an empty :class:`scrapy.http.headers.Headers` instance (not
+ ``None``), and ``request.flags = []`` remains an empty list instead of
+ being set to ``None``. Update code that relied on ``is None`` checks or
+ the previous coercion behaviour.
+
+ (:issue:`7036`, :issue:`7367`, :issue:`7374`)
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+- The context factory class set as the value of the
+ ``DOWNLOADER_CLIENTCONTEXTFACTORY`` setting is now required to support the
+ ``method`` argument of ``__init__()``, recommended since Scrapy 1.2.0.
+ (:issue:`7353`)
+
+Deprecations
+~~~~~~~~~~~~
+
+- ``scrapy.mail.MailSender`` is deprecated. Please use :mod:`smtplib`,
+ :mod:`twisted.mail.smtp` or other 3rd party email libraries.
+ (:issue:`7249`, :issue:`7263`)
+
+- The ``scrapy.extensions.statsmailer.StatsMailer`` extension is deprecated.
+ You can instead implement your own notifications by handling the
+ :signal:`spider_closed` signal.
+ (:issue:`7249`, :issue:`7263`)
+
+- The ``MEMUSAGE_NOTIFY_MAIL`` setting is deprecated. You can instead
+ implement your own notifications by handling the
+ :signal:`memusage_warning_reached` and :signal:`spider_closed` signals.
+ (:issue:`7249`, :issue:`7263`)
+
+- The ``DNS_RESOLVER`` setting was renamed to :setting:`TWISTED_DNS_RESOLVER`
+ and the old name is deprecated.
+ (:issue:`7350`, :issue:`7361`)
+
+- The ``DOWNLOADER_CLIENTCONTEXTFACTORY`` setting is deprecated. If you were
+ using it to switch to
+ ``scrapy.core.downloader.contextfactory.BrowserLikeContextFactory``, please
+ use the new :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting instead. If you
+ cannot use the default context factory for some other reason, please
+ subclass the :ref:`download handler ` instead.
+ (:issue:`7352`, :issue:`7379`)
+
+- ``scrapy.core.downloader.contextfactory.BrowserLikeContextFactory`` is
+ deprecated. You can set the new :setting:`DOWNLOAD_VERIFY_CERTIFICATES`
+ setting to ``True`` instead.
+ (:issue:`7379`)
+
+- The following implementation details of the context factory handling code
+ are deprecated:
+
+ - ``scrapy.core.downloader.contextfactory.AcceptableProtocolsContextFactory``
+
+ - ``scrapy.core.downloader.contextfactory.load_context_factory_from_settings()``
+
+ - ``scrapy.core.downloader.contextfactory.ScrapyClientContextFactory``
+
+ - ``scrapy.core.downloader.tls.ScrapyClientTLSOptions``
+
+ (:issue:`7353`, :issue:`7391`)
+
+- Passing :class:`str` instead of :class:`bytes` to
+ :class:`scrapy.utils.sitemap.Sitemap` and
+ :func:`scrapy.utils.sitemap.sitemap_urls_from_robots` is deprecated.
+ (:issue:`7007`)
+
+- ``scrapy.utils.misc.walk_modules()`` is deprecated. You can use
+ :func:`scrapy.utils.misc.walk_modules_iter` instead.
+ (:issue:`7388`)
+
+- ``scrapy.shell.Shell.inthread`` is deprecated. You can use
+ :attr:`scrapy.shell.Shell.fetch_available` instead to check if
+ :func:`~scrapy.shell.Shell.fetch` can be used.
+ (:issue:`7395`)
+
+- ``scrapy.commands.ScrapyCommand.set_crawler()`` is deprecated.
+ (:issue:`7276`)
+
+New features
+~~~~~~~~~~~~
+
+- Added an *experimental* mode for running Scrapy without installing a
+ Twisted reactor: set :setting:`TWISTED_REACTOR_ENABLED` to ``False`` to
+ enable it. This mode has limitations, refer to :ref:`its documentation
+ ` for details. As long as it's experimental, its
+ behavior and related features and APIs may change in future Scrapy releases
+ in a breaking way.
+ (:issue:`6219`,
+ :issue:`7185`,
+ :issue:`7186`,
+ :issue:`7187`,
+ :issue:`7188`,
+ :issue:`7190`,
+ :issue:`7197`,
+ :issue:`7199`,
+ :issue:`7209`,
+ :issue:`7228`,
+ :issue:`7355`,
+ :issue:`7366`,
+ :issue:`7385`,
+ :issue:`7395`)
+
+- Added the :func:`scrapy.utils.reactorless.is_reactorless` function that
+ checks if there is a running asyncio event loop but no Twisted reactor.
+ (:issue:`7185`, :issue:`7199`)
+
+- Changed :func:`scrapy.utils.asyncio.is_asyncio_available` to return
+ ``True`` if there is a running asyncio loop, even if no Twisted reactor is
+ installed.
+ (:issue:`7185`, :issue:`7199`)
+
+- Added an *experimental* download handler that uses the httpx_ library and
+ doesn't require a Twisted reactor:
+ :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. As
+ long as it's experimental, its behavior may change in future Scrapy
+ releases in a breaking way.
+ (:issue:`6805`, :issue:`7239`, :issue:`7368`, :issue:`7384`)
+
+ .. _httpx: https://www.python-httpx.org/
+
+- Added the :setting:`DOWNLOAD_BIND_ADDRESS` setting as a global counterpart
+ to the per-request :reqmeta:`bindaddress` meta key.
+ (:issue:`7266`, :issue:`7283`)
+
+- Added the :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting that can be set
+ to ``True`` to make Scrapy abort HTTPS requests when the server certificate
+ is invalid or doesn't match the domain.
+ (:issue:`7379`)
+
+- The built-in HTTP :ref:`download handlers ` now
+ raise Scrapy-specific exceptions instead of implementation-specific ones,
+ to allow unified handling of similar problems caused by different
+ implementations. The default value of the :setting:`RETRY_EXCEPTIONS`
+ setting was updated replacing Twisted-specific exceptions with these new
+ ones. The exceptions:
+
+ - :exc:`~scrapy.exceptions.CannotResolveHostError`
+
+ - :exc:`~scrapy.exceptions.DownloadCancelledError`
+
+ - :exc:`~scrapy.exceptions.DownloadConnectionRefusedError`
+
+ - :exc:`~scrapy.exceptions.DownloadFailedError`
+
+ - :exc:`~scrapy.exceptions.DownloadTimeoutError`
+
+ - :exc:`~scrapy.exceptions.ResponseDataLossError`
+
+ - :exc:`~scrapy.exceptions.UnsupportedURLSchemeError`
+
+ (:issue:`7208`)
+
+- Added the :signal:`memusage_warning_reached` signal emitted by the
+ :class:`~scrapy.extensions.memusage.MemoryUsage` extension when the memory
+ usage reaches :setting:`MEMUSAGE_WARNING_MB`.
+ (:issue:`7249`, :issue:`7263`)
+
+- Added
+ :meth:`Headers.to_tuple_list() `
+ that returns headers as a list of ``(key, value)`` tuples.
+ (:issue:`7239`)
+
+- :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler` now uses the
+ download handler configured for the ``"https"`` scheme to make requests
+ instead of always using
+ :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`.
+ (:issue:`7369`, :issue:`7370`)
+
+- Added :func:`scrapy.utils.misc.walk_modules_iter` as a replacement for
+ ``scrapy.utils.misc.walk_modules()`` that returns an iterable instead of a
+ list.
+ (:issue:`7388`)
+
+Improvements
+~~~~~~~~~~~~
+
+- :func:`asyncio.to_thread` is now used instead of
+ :func:`twisted.internet.threads.deferToThread` in the built-in feed
+ storages, media pipeline storages and the
+ :func:`scrapy.utils.decorators.inthread` decorator when available.
+ (:issue:`7183`, :issue:`7184`, :issue:`7349`)
+
+- Improved memory footprint of :class:`~scrapy.Request` and
+ :class:`~scrapy.http.Response` objects by adding ``__slots__`` and omitting
+ empty lists and dicts in some internal attributes.
+ (:issue:`7036`, :issue:`7367`, :issue:`7374`)
+
+- :class:`~scrapy.core.downloader.contextfactory._ScrapyClientContextFactory`
+ no longer mutates the SSL context, to avoid the behavior that was
+ deprecated in pyOpenSSL 25.1.0.
+ (:issue:`6859`, :issue:`7353`)
+
+- Improved memory usage of :class:`~scrapy.spiders.sitemap.SitemapSpider` and
+ :class:`scrapy.utils.sitemap.Sitemap`.
+ (:issue:`3529`, :issue:`7007`)
+
+- Improved the scheduling behavior of
+ :class:`~scrapy.pqueues.DownloaderAwarePriorityQueue` when crawling
+ multiple domains.
+ (:issue:`7293`, :issue:`7351`)
+
+- :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` and
+ :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now handle
+ TLS verbose logging (see :setting:`DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING`)
+ directly instead of relying on
+ :class:`~scrapy.core.downloader.contextfactory._ScrapyClientContextFactory`.
+ (:issue:`7387`)
+
+- The server certificate verification code now correctly handles certificates
+ with IP addresses in ``subjectAltName``.
+ (:issue:`7353`)
+
+- Improved reliability of :func:`scrapy.utils.trackref.get_oldest`.
+ (:issue:`1758`, :issue:`7375`)
+
+- Other code refactoring and improvements.
+ (:issue:`7210`, :issue:`7238`, :issue:`7376`, :issue:`7386`, :issue:`7395`,
+ :issue:`7405`, :issue:`7410`)
+
+Bug fixes
+~~~~~~~~~
+
+- :ref:`Media pipelines ` should now wait for uploads
+ to asynchronous storages (e.g.
+ :class:`~scrapy.pipelines.files.S3FilesStore`) to complete.
+ (:issue:`2183`, :issue:`6369`, :issue:`7182`)
+
+- Fixed merging ``*_BASE`` settings (e.g. merging
+ :setting:`DOWNLOADER_MIDDLEWARES` with
+ :setting:`DOWNLOADER_MIDDLEWARES_BASE`) when a component is referred to by
+ a class object in one setting and by a string import path in the other one.
+ (:issue:`6912`, :issue:`6993`)
+
+- ``scrapy runspider`` and ``scrapy crawl`` now set the exit code to 1 if an
+ exception happened early (this was broken since Scrapy 2.13.0).
+ (:issue:`6820`, :issue:`7255`)
+
+- Fixed repeated warnings about data loss (see
+ :setting:`DOWNLOAD_FAIL_ON_DATALOSS`) not being suppressed in
+ :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`.
+ (:issue:`7222`)
+
+- Improved FTP connection management in
+ :class:`scrapy.pipelines.files.FTPFilesStore`.
+ (:issue:`7256`)
+
+- Fixed the ``spider`` variable in the :ref:`shell `, which
+ wasn't available since Scrapy 2.13.0.
+ (:issue:`7395`)
+
+Documentation
+~~~~~~~~~~~~~
+
+- The ``llms.txt`` and ``llms-full.txt`` files and Markdown versions of pages
+ are now generated when the HTML documentation is built.
+ (:issue:`7380`)
+
+- Added a "Copy as Markdown" button to the HTML documentation.
+ (:issue:`7380`)
+
+- Added :ref:`docs for using Pydantic models as items `.
+ (:issue:`6955`, :issue:`6966`)
+
+- Documented :ref:`job directory contents `.
+ (:issue:`4842`, :issue:`5260`)
+
+- Improved docs for :attr:`~scrapy.Request.dont_filter`.
+ (:issue:`6398`, :issue:`7245`)
+
+- Clarified that settings related to :setting:`TWISTED_DNS_RESOLVER` are only
+ taken into account if the selected resolver supports them.
+ (:issue:`7385`)
+
+- Other documentation improvements and fixes.
+ (:issue:`7248`, :issue:`7274`, :issue:`7406`, :issue:`7408`)
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+- Added the ``no-reactor`` test environment that doesn't install a Twisted
+ reactor and uses ``pytest-asyncio`` instead of ``pytest-twisted`` to run
+ asynchronous test functions.
+ (:issue:`6952`, :issue:`7189`, :issue:`7233`, :issue:`7234`, :issue:`7254`,
+ :issue:`7259`)
+
+- Fixed running tests with ``pytest-xdist``.
+ (:issue:`7216`, :issue:`7257`)
+
+- Type hints improvements and fixes.
+ (:issue:`7300`, :issue:`7331`)
+
+- CI and test improvements and fixes.
+ (:issue:`7060`,
+ :issue:`7223`,
+ :issue:`7232`,
+ :issue:`7241`,
+ :issue:`7250`,
+ :issue:`7256`,
+ :issue:`7276`,
+ :issue:`7277`,
+ :issue:`7279`,
+ :issue:`7329`,
+ :issue:`7363`,
+ :issue:`7381`,
+ :issue:`7402`)
+
+.. _release-2.14.2:
+
+Scrapy 2.14.2 (2026-03-12)
+--------------------------
+
+Security bug fixes
+~~~~~~~~~~~~~~~~~~
+
+- Values from the ``Referrer-Policy`` header of HTTP responses are no longer
+ executed as Python callables. See the `cwxj-rr6w-m6w7`_ security advisory
+ for details.
+
+ .. _cwxj-rr6w-m6w7: https://github.com/scrapy/scrapy/security/advisories/GHSA-cwxj-rr6w-m6w7
+
+- In line with the `standard
+ `__, 301 redirects of
+ ``POST`` requests are converted into ``GET`` requests.
+
+ Converting to a ``GET`` request implies not only a method change, but also
+ omitting the body and ``Content-*`` headers in the redirect request. On
+ cross-origin redirects (for example, cross-domain redirects), this is
+ effectively a security bug fix for scenarios where the body contains
+ secrets.
+
+Deprecations
+~~~~~~~~~~~~
+
+- Passing a response URL string as the first positional argument to
+ :meth:`scrapy.spidermiddlewares.referer.RefererMiddleware.policy` is
+ deprecated. Pass a :class:`~scrapy.http.Response` instead.
+
+ The parameter has also been renamed to ``response`` to reflect this change.
+ The old parameter name (``resp_or_url``) is deprecated.
+
+New features
+~~~~~~~~~~~~
+
+- Added a new setting, :setting:`REFERRER_POLICIES`, to allow customizing
+ supported referrer policies.
+
+Bug fixes
+~~~~~~~~~
+
+- Made additional redirect scenarios convert to ``GET`` in line with the
+ `standard `__:
+
+ - Only ``POST`` 302 redirects are converted into ``GET`` requests; other
+ methods are preserved.
+
+ - ``HEAD`` 303 redirects are not converted into ``GET`` requests.
+
+ - ``GET`` 303 redirects do not have their body or standard ``Content-*``
+ headers removed.
+
+- Redirects where the original request body is dropped now also have their
+ ``Content-Encoding``, ``Content-Language`` and ``Content-Location`` headers
+ removed, in addition to the ``Content-Type`` and ``Content-Length`` headers
+ that were already being removed.
+
+- Redirects now preserve the source URL fragment if the redirect URL does not
+ include one. This is useful when using browser-based download handlers,
+ such as `scrapy-playwright`_ or `scrapy-zyte-api`_, while letting Scrapy
+ handle redirects.
+
+ .. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright
+ .. _scrapy-zyte-api: https://scrapy-zyte-api.readthedocs.io/en/latest/
+
+- The ``Referer`` header is now removed on redirect if
+ :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` is disabled.
+
+- The handling of the ``Referer`` header on redirects now takes into account
+ the ``Referer-Policy`` header of the response that triggers the redirect.
+
+.. _release-2.14.1:
+
+Scrapy 2.14.1 (2026-01-12)
+--------------------------
+
+Deprecations
+~~~~~~~~~~~~
+
+- ``scrapy.utils.defer.maybeDeferred_coro()`` is deprecated. (:issue:`7212`)
+
+Bug fixes
+~~~~~~~~~
+
+- Fixed custom stats collectors that require a ``spider`` argument in their
+ ``open_spider()`` and ``close_spider()`` methods not receiving the
+ argument when called by the engine.
+
+ Note, however, that the ``spider`` argument is now deprecated and will stop
+ being passed in a future version of Scrapy.
+
+ (:issue:`7213`)
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+- Replaced deprecated ``codecov/test-results-action@v1`` GitHub Action with
+ ``codecov/codecov-action@v5``.
+ (:issue:`7180`, :issue:`7215`)
+
+.. _release-2.14.0:
+
+Scrapy 2.14.0 (2026-01-05)
+--------------------------
+
+Highlights:
+
+- More coroutine-based replacements for Deferred-based APIs
+
+- The default priority queue is now ``DownloaderAwarePriorityQueue``
+
+- Dropped support for Python 3.9 and PyPy 3.10
+
+- Improved and documented the API for custom download handlers
+
+Modified requirements
+~~~~~~~~~~~~~~~~~~~~~
+
+- Dropped support for Python 3.9.
+ (:issue:`7121`)
+
+- Dropped support for PyPy 3.10.
+ (:issue:`7050`)
+
+- Increased the minimum versions of the following dependencies:
+
+ - lxml_: 4.6.0 → 4.6.4
+
+ - Pillow_ (optional dependency): 8.0.0 → 8.3.2
+
+ - botocore_ (optional dependency): 1.4.87 → 1.13.45
+
+- Restored support for ``brotlicffi`` dropped in Scrapy 2.13.4. Its minimum
+ supported version is now ``1.2.0.0``.
+ (:issue:`7160`)
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- If you set the :setting:`TWISTED_REACTOR` setting to a :ref:`non-asyncio
+ value ` at the :ref:`spider level `, you
+ may now need to set the :setting:`FORCE_CRAWLER_PROCESS` setting to
+ ``True`` when running Scrapy via :ref:`its command-line tool
+ ` to avoid a reactor mismatch exception.
+ (:issue:`6845`)
+
+- The ``log_count/*`` stats no longer count some of the early messages that
+ they counted before. While the earliest log messages, emitted before the
+ counter is initialized, were never counted, the counter initialization now
+ happens later than in previous Scrapy versions. You may need to adjust
+ expected values if you retrieve and compare values of these stats in your
+ code.
+ (:issue:`7046`)
+
+- The classes listed below are now :term:`abstract base classes `. They cannot be instantiated directly and their subclasses
+ need to override the abstract methods listed below to be able to be
+ instantiated. If you previously instantiated these classes directly, you
+ will now need to subclass them and provide trivial (e.g. empty)
+ implementations for the abstract methods.
+
+ - :class:`scrapy.commands.ScrapyCommand`
+
+ - :meth:`~scrapy.commands.ScrapyCommand.run`
+
+ - :meth:`~scrapy.commands.ScrapyCommand.short_desc`
+
+ - :class:`scrapy.exporters.BaseItemExporter`
+
+ - :meth:`~scrapy.exporters.BaseItemExporter.export_item`
+
+ - :class:`scrapy.extensions.feedexport.BlockingFeedStorage`
+
+ - :meth:`~scrapy.extensions.feedexport.BlockingFeedStorage._store_in_thread`
+
+ - :class:`scrapy.middleware.MiddlewareManager`
+
+ - :meth:`~scrapy.middleware.MiddlewareManager._get_mwlist_from_settings`
+
+ - :class:`scrapy.spidermiddlewares.referer.ReferrerPolicy`
+
+ - :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer`
+
+ (:issue:`6930`)
+
+- Scrapy no longer passes a ``spider`` argument to any methods of the
+ :setting:`stats collector `. It wasn't passed in many of the
+ calls even in older Scrapy versions, so we don't expect existing custom
+ stats collector implementations to require a ``spider`` argument. If your
+ implementation needs a :class:`~scrapy.Spider` instance, you can get it
+ from the :class:`~scrapy.crawler.Crawler` instance passed to the
+ constructor.
+ (:issue:`7011`)
+
+- :class:`scrapy.middleware.MiddlewareManager` no longer includes code for
+ handling ``open_spider()`` and ``close_spider()`` component methods. As
+ this code was only used for pipelines it was moved into
+ :class:`scrapy.pipelines.ItemPipelineManager`. This change should only
+ affect custom subclasses of :class:`~scrapy.middleware.MiddlewareManager`.
+ The following code was moved:
+
+ - ``scrapy.middleware.MiddlewareManager.open_spider()``
+
+ - ``scrapy.middleware.MiddlewareManager.close_spider()``
+
+ - Code in ``scrapy.middleware.MiddlewareManager._add_middleware()`` that
+ processes ``open_spider()`` and ``close_spider()`` component methods.
+
+ (:issue:`7006`)
+
+- :meth:`scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware.process_request`
+ now returns a coroutine, previously it returned a
+ :class:`~twisted.internet.defer.Deferred` object or ``None``. The
+ ``robot_parser()`` method was also changed to return a coroutine. This
+ change only impacts code that subclasses
+ :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` or
+ calls its methods directly.
+ (:issue:`6802`)
+
+- The built-in :ref:`download handlers ` have been
+ refactored, changing the signatures of their methods. This change should
+ only affect user code that subclasses any of these handlers or calls their
+ methods directly.
+ (:issue:`6778`, :issue:`7164`)
+
+- :meth:`scrapy.pipelines.media.MediaPipeline.process_item` now returns a
+ coroutine, previously it returned a
+ :class:`~twisted.internet.defer.Deferred` object. This
+ change only impacts code that calls this method directly.
+ (:issue:`7177`)
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+- The ``from_settings()`` method of the following components, deprecated in
+ Scrapy 2.12.0, is removed. You should use ``from_crawler()`` instead.
+
+ - :class:`scrapy.dupefilters.RFPDupeFilter`
+ - :class:`scrapy.mail.MailSender`
+ - :class:`scrapy.middleware.MiddlewareManager`
+ - :class:`scrapy.core.downloader.contextfactory.ScrapyClientContextFactory`
+ - :class:`scrapy.pipelines.files.FilesPipeline`
+ - :class:`scrapy.pipelines.images.ImagesPipeline`
+
+ (:issue:`7126`)
+
+- Scrapy no longer calls ``from_settings()`` methods of 3rd-party
+ :ref:`components `, deprecated in Scrapy 2.12.0. You
+ should define a ``from_crawler()`` method instead.
+ (:issue:`7126`)
+
+- The initialization flow of :class:`scrapy.pipelines.media.MediaPipeline`
+ and its subclasses was simplified, it now mandates ``from_crawler()``
+ methods and ``crawler`` arguments of ``__init__()`` methods. Not using
+ these was deprecated in Scrapy 2.12.0.
+ (:issue:`7126`)
+
+- The ``REQUEST_FINGERPRINTER_IMPLEMENTATION`` setting, deprecated in Scrapy
+ 2.12.0, is removed.
+ (:issue:`7126`)
+
+- The ``scrapy.utils.misc.create_instance()`` function, deprecated in Scrapy
+ 2.12.0, is removed. Use :func:`scrapy.utils.misc.build_from_crawler`
+ instead.
+ (:issue:`7126`)
+
+- The ``scrapy.core.downloader.Downloader._get_slot_key()`` function,
+ deprecated in Scrapy 2.12.0, is removed. Use
+ :meth:`scrapy.core.downloader.Downloader.get_slot_key` instead.
+ (:issue:`7126`)
+
+- The ``scrapy.twisted_version`` attribute, deprecated in Scrapy 2.12.0, is
+ removed. You should instead use the :attr:`twisted.version` attribute
+ directly.
+ (:issue:`7126`)
+
+- The following utility functions, deprecated in Scrapy 2.12.0, are removed:
+
+ - ``scrapy.utils.defer.process_chain_both()``
+ - ``scrapy.utils.python.equal_attributes()``
+ - ``scrapy.utils.python.flatten()``
+ - ``scrapy.utils.python.iflatten()``
+ - ``scrapy.utils.request.request_authenticate()``
+ - ``scrapy.utils.test.assert_samelines()``
+
+ (:issue:`7126`)
+
+- ``scrapy.utils.serialize.ScrapyJSONDecoder``, deprecated in Scrapy 2.12.0,
+ is removed.
+ (:issue:`7126`)
+
+- The ``scrapy.extensions.feedexport.build_storage()`` function, deprecated
+ in Scrapy 2.12.0, is removed, you can instead call the builder callable
+ directly.
+ (:issue:`7126`)
+
+- ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware``, deprecated in
+ Scrapy 2.11.2, is removed.
+ :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware` should be
+ used instead.
+ (:issue:`6926`)
+
+Deprecations
+~~~~~~~~~~~~
+
+- The following methods that return a
+ :class:`~twisted.internet.defer.Deferred` are deprecated in favor of their
+ coroutine-based replacements:
+
+ - :class:`scrapy.core.downloader.handlers.DownloadHandlers`
+
+ - ``download_request()`` (use
+ :meth:`~scrapy.core.downloader.handlers.DownloadHandlers.download_request_async`)
+
+ - :class:`scrapy.core.downloader.middleware.DownloaderMiddlewareManager`
+
+ - ``download()`` (use
+ :meth:`~scrapy.core.downloader.middleware.DownloaderMiddlewareManager.download_async`)
+
+ - :class:`scrapy.core.engine.ExecutionEngine`
+
+ - ``start()`` (use
+ :meth:`~scrapy.core.engine.ExecutionEngine.start_async`)
+
+ - ``stop()`` (use
+ :meth:`~scrapy.core.engine.ExecutionEngine.stop_async`)
+
+ - ``close()`` (use
+ :meth:`~scrapy.core.engine.ExecutionEngine.close_async`)
+
+ - ``open_spider()`` (use
+ :meth:`~scrapy.core.engine.ExecutionEngine.open_spider_async`)
+
+ - ``close_spider()`` (use
+ :meth:`~scrapy.core.engine.ExecutionEngine.close_spider_async`)
+
+ - ``download()`` (use
+ :meth:`~scrapy.core.engine.ExecutionEngine.download_async`)
+
+ - :class:`scrapy.core.scraper.Scraper`
+
+ - ``open_spider()`` (use
+ :meth:`~scrapy.core.scraper.Scraper.open_spider_async`)
+
+ - ``call_spider()`` (use
+ :meth:`~scrapy.core.scraper.Scraper.call_spider_async`)
+
+ - ``close_spider()`` (use
+ :meth:`~scrapy.core.scraper.Scraper.close_spider_async`)
+
+ - ``handle_spider_output()`` (use
+ :meth:`~scrapy.core.scraper.Scraper.handle_spider_output_async`)
+
+ - ``start_itemproc()`` (use
+ :meth:`~scrapy.core.scraper.Scraper.start_itemproc_async`)
+
+ - :class:`scrapy.core.spidermw.SpiderMiddlewareManager`
+
+ - ``scrape_response()`` (use
+ :meth:`~scrapy.core.spidermw.SpiderMiddlewareManager.scrape_response_async`)
+
+ - :class:`scrapy.crawler.Crawler`
+
+ - ``stop()`` (use :meth:`~scrapy.crawler.Crawler.stop_async`)
+
+ - :class:`scrapy.pipelines.ItemPipelineManager`
+
+ - ``process_item()`` (use
+ :meth:`~scrapy.pipelines.ItemPipelineManager.process_item_async`)
+
+ - ``open_spider()`` (use
+ :meth:`~scrapy.pipelines.ItemPipelineManager.open_spider_async`)
+
+ - ``close_spider()`` (use
+ :meth:`~scrapy.pipelines.ItemPipelineManager.close_spider_async`)
+
+ - :class:`scrapy.signalmanager.SignalManager`
+
+ - ``send_catch_log_deferred()`` (use
+ :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_async`)
+
+ - ``scrapy.utils.signal.send_catch_log_deferred()`` (use
+ :func:`scrapy.utils.signal.send_catch_log_async`)
+
+ (:issue:`6791`, :issue:`6842`, :issue:`6979`, :issue:`6997`, :issue:`6999`,
+ :issue:`7005`, :issue:`7043`, :issue:`7069`, :issue:`7161`, :issue:`7164`)
+
+- The following spider attributes are deprecated in favor of settings:
+
+ - ``download_maxsize`` (use :setting:`DOWNLOAD_MAXSIZE`)
+
+ - ``download_timeout`` (use :setting:`DOWNLOAD_TIMEOUT`)
+
+ - ``download_warnsize`` (use :setting:`DOWNLOAD_WARNSIZE`)
+
+ - ``max_concurrent_requests`` (use
+ :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`)
+
+ - ``user_agent`` (use :setting:`USER_AGENT`)
+
+ (:issue:`6988`, :issue:`6994`, :issue:`7038`, :issue:`7039`, :issue:`7117`,
+ :issue:`7176`)
+
+- Returning a :class:`~twisted.internet.defer.Deferred` from the following
+ user-defined functions is deprecated in favor of defining them as coroutine
+ functions:
+
+ - spider callbacks and errbacks (which was never officially supported and
+ may work incorrectly)
+
+ - the ``process_request()``, ``process_response()`` and
+ ``process_exception()`` methods of custom downloader middlewares
+
+ - the ``process_item()``, ``open_spider()`` and ``close_spider()`` methods
+ of custom pipelines
+
+ - signal handlers
+
+ - the ``download_request()`` and ``close()`` methods of custom download
+ handlers
+
+ (:issue:`6718`, :issue:`6778`, :issue:`7069`, :issue:`7147`, :issue:`7148`,
+ :issue:`7149`, :issue:`7150`, :issue:`7151`, :issue:`7161`, :issue:`7164`,
+ :issue:`7179`)
+
+- Passing a ``spider`` argument to the following methods is deprecated:
+
+ - :meth:`scrapy.core.spidermw.SpiderMiddlewareManager.process_start`
+
+ - :meth:`scrapy.core.downloader.Downloader.fetch`
+
+ - :meth:`scrapy.core.downloader.Downloader._get_slot`
+
+ - :meth:`scrapy.core.downloader.handlers.DownloadHandlers.download_request`
+
+ - all public methods of :class:`scrapy.statscollectors.StatsCollector`
+
+ - :meth:`scrapy.spidermiddlewares.base.BaseSpiderMiddleware.process_spider_output`
+
+ - :meth:`scrapy.spidermiddlewares.base.BaseSpiderMiddleware.process_spider_output_async`
+
+ - all ``process_*()`` methods of built-in downloader middlewares
+
+ - all ``process_*()`` methods of built-in spider middlewares
+
+ - :meth:`scrapy.pipelines.media.MediaPipeline.open_spider`
+
+ - :meth:`scrapy.pipelines.media.MediaPipeline.process_item`
+
+ (:issue:`6750`, :issue:`6927`, :issue:`6984`, :issue:`7006`, :issue:`7011`,
+ :issue:`7033`, :issue:`7037`, :issue:`7045`, :issue:`7178`)
+
+- Instantiating subclasses of :class:`scrapy.middleware.MiddlewareManager`
+ without a :class:`~scrapy.crawler.Crawler` instance is deprecated.
+ (:issue:`6984`)
+
+- For the following user-defined functions and methods requiring a ``spider``
+ argument is deprecated, if you need a :class:`~scrapy.Spider` instance
+ inside them you should get it from the :class:`~scrapy.crawler.Crawler`
+ instance (you may need to refactor your code to save that instance in e.g.
+ the ``from_crawler()`` method):
+
+ - the ``process_request()``, ``process_response()`` and
+ ``process_exception()`` methods of custom downloader middlewares
+
+ - the ``process_spider_input()``, ``process_spider_output()``,
+ ``process_spider_output_async()`` and ``process_spider_exception()``
+ methods of custom spider middlewares
+
+ - the ``process_item()`` method of custom pipelines
+
+ - the ``fetch()`` method of a custom :setting:`DOWNLOADER`
+
+ (:issue:`6927`, :issue:`6984`, :issue:`7006`, :issue:`7037`)
+
+- The following things in custom download handlers are deprecated:
+
+ - not having a ``lazy`` attribute (you should define it as ``True`` if you
+ want to keep the current behavior)
+
+ - returning a :class:`~twisted.internet.defer.Deferred` from the
+ ``download_request()`` method (you should refactor it to return a
+ coroutine; you also need to remove the ``spider`` argument when doing
+ this)
+
+ - not having a ``close()`` method, having a synchronous one or one that
+ returns a :class:`~twisted.internet.defer.Deferred` (you should refactor
+ it to return a coroutine or add an empty one if you don't have it)
+
+ (:issue:`6778`, :issue:`7164`)
+
+- Custom implementations of :setting:`ITEM_PROCESSOR` should now define
+ ``process_item_async()``, ``open_spider_async()`` and
+ ``close_spider_async()`` methods instead of, or in addition to,
+ ``process_item()``, ``open_spider()`` and ``close_spider()``.
+ (:issue:`7005`, :issue:`7043`)
+
+- The ``CONCURRENT_REQUESTS_PER_IP`` setting is deprecated, use
+ :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` instead.
+ (:issue:`6917`, :issue:`6921`)
+
+- The ``scrapy.core.downloader.handlers.http`` module is deprecated. You
+ should import
+ :class:`scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
+ directly instead of importing the
+ ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler`` alias.
+ (:issue:`7079`)
+
+- The ``scrapy.utils.decorators.defers()`` decorator is deprecated, you can
+ use :func:`twisted.internet.defer.maybeDeferred` directly or reimplement
+ this decorator in your code.
+ (:issue:`7164`)
+
+- ``scrapy.spiders.CrawlSpider._parse_response()`` is deprecated, use
+ :meth:`scrapy.spiders.CrawlSpider.parse_with_rules` instead.
+ (:issue:`4463`, :issue:`6804`)
+
+- The functions that add a delay to a Deferred are deprecated, their
+ underlying Twisted functions can be used instead, either directly if a
+ delay isn't needed, or with some explicit way to add a delay if it's
+ needed:
+
+ - ``scrapy.utils.defer.mustbe_deferred()`` (you can use
+ :func:`twisted.internet.defer.maybeDeferred`)
+
+ - ``scrapy.utils.defer.defer_succeed()`` (you can use
+ :func:`twisted.internet.defer.succeed`)
+
+ - ``scrapy.utils.defer.defer_fail()`` (you can use
+ :func:`twisted.internet.defer.fail`)
+
+ - ``scrapy.utils.defer.defer_result()`` (you can use
+ :func:`twisted.internet.defer.succeed` and
+ :func:`twisted.internet.defer.fail`)
+
+ (:issue:`6937`)
+
+New features
+~~~~~~~~~~~~
+
+- Added :class:`scrapy.crawler.AsyncCrawlerProcess` and
+ :class:`scrapy.crawler.AsyncCrawlerRunner` as counterparts to
+ :class:`~scrapy.crawler.CrawlerProcess` and
+ :class:`~scrapy.crawler.CrawlerRunner` that offer coroutine-based APIs.
+ (:issue:`6789`, :issue:`6790`, :issue:`6796`, :issue:`6817`, :issue:`6845`,
+ :issue:`7034`)
+
+- Added coroutine counterparts to some of the Deferred-based APIs:
+
+ - :class:`scrapy.core.downloader.handlers.DownloadHandlers`
+
+ - :meth:`~scrapy.core.downloader.handlers.DownloadHandlers.download_request_async`
+ (to ``download_request()``)
+
+ - :class:`scrapy.core.downloader.middleware.DownloaderMiddlewareManager`
+
+ - :meth:`~scrapy.core.downloader.middleware.DownloaderMiddlewareManager.download_async`
+ (to ``download()``)
+
+ - :class:`scrapy.core.engine.ExecutionEngine`
+
+ - :meth:`~scrapy.core.engine.ExecutionEngine.start_async` (to
+ ``start()``)
+
+ - :meth:`~scrapy.core.engine.ExecutionEngine.stop_async` (to
+ ``stop()``)
+
+ - :meth:`~scrapy.core.engine.ExecutionEngine.close_async` (to
+ ``close()``)
+
+ - :meth:`~scrapy.core.engine.ExecutionEngine.open_spider_async` (to
+ ``open_spider()``)
+
+ - :meth:`~scrapy.core.engine.ExecutionEngine.close_spider_async` (to
+ ``close_spider()``)
+
+ - :meth:`~scrapy.core.engine.ExecutionEngine.download_async` (to
+ ``download()``)
+
+ - :class:`scrapy.core.scraper.Scraper`
+
+ - :meth:`~scrapy.core.scraper.Scraper.open_spider_async` (to
+ ``open_spider()``)
+
+ - :meth:`~scrapy.core.scraper.Scraper.close_spider_async` (to
+ ``close_spider()``)
+
+ - :meth:`~scrapy.core.scraper.Scraper.start_itemproc_async` (to
+ ``start_itemproc()``)
+
+ - :class:`scrapy.crawler.Crawler`
+
+ - :meth:`~scrapy.crawler.Crawler.crawl_async` (to ``crawl()``)
+
+ - :meth:`~scrapy.crawler.Crawler.stop_async` (to ``stop()``)
+
+ - :class:`scrapy.pipelines.ItemPipelineManager`
+
+ - :meth:`~scrapy.pipelines.ItemPipelineManager.process_item_async` (to
+ ``process_item()``)
+
+ - :meth:`~scrapy.pipelines.ItemPipelineManager.open_spider_async` (to
+ ``open_spider()``)
+
+ - :meth:`~scrapy.pipelines.ItemPipelineManager.close_spider_async` (to
+ ``close_spider()``)
+
+ - :class:`scrapy.signalmanager.SignalManager`
+
+ - :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_async` (to
+ ``send_catch_log_deferred()``)
+
+ (:issue:`6781`, :issue:`6791`, :issue:`6792`, :issue:`6795`, :issue:`6801`,
+ :issue:`6817`, :issue:`6842`, :issue:`6997`, :issue:`7005`, :issue:`7043`,
+ :issue:`7069`,:issue:`7164`, :issue:`7202`)
+
+- The default value of the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is now
+ ``'scrapy.pqueues.DownloaderAwarePriorityQueue'``.
+ (:issue:`6924`, :issue:`6940`)
+
+- Added :class:`scrapy.extensions.logcount.LogCount`, an enabled-by-default
+ extension that is responsible for the ``log_count/*`` stats. Previously,
+ this code was in :class:`scrapy.crawler.Crawler` and couldn't be disabled.
+ (:issue:`7046`)
+
+- Added :meth:`scrapy.spiders.CrawlSpider.parse_with_rules` as a public
+ replacement for ``_parse_response()``.
+ (:issue:`4463`, :issue:`6804`)
+
+- Added :func:`scrapy.utils.asyncio.is_asyncio_available` as an alternative
+ to :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` with a
+ future-proof name and semantics.
+ (:issue:`6827`)
+
+- The API for :ref:`download handlers `, previously
+ undocumented, has been modernized and documented. An optional base class,
+ :class:`scrapy.core.downloader.handlers.base.BaseDownloadHandler`, has been
+ added to simplify writing custom download handlers that conform to the
+ current API.
+ (:issue:`4944`, :issue:`6778`, :issue:`7164`)
+
+- Added :func:`scrapy.utils.defer.ensure_awaitable`, which can be helpful to
+ call user-defined functions that can return coroutines, Deferreds or
+ values directly.
+ (:issue:`7005`)
+
+- The ``requests.seen`` file, written by
+ :class:`~scrapy.dupefilters.RFPDupeFilter` when :ref:`job persistence
+ ` is enabled, now uses line buffering to reduce data loss in
+ spider crashes.
+ (:issue:`6019`, :issue:`7094`)
+
+- Images downloaded by :class:`~scrapy.pipelines.images.ImagesPipeline` are
+ now automatically transposed based on EXIF data.
+ (:issue:`6525`, :issue:`6975`)
+
+Improvements
+~~~~~~~~~~~~
+
+- Refactored internal functions to use coroutines instead of Deferreds.
+ (:issue:`6795`, :issue:`6852`, :issue:`6855`, :issue:`6858`, :issue:`7159`)
+
+- Commands that don't need a :class:`~scrapy.crawler.CrawlerProcess` instance
+ no longer create it.
+ (:issue:`6824`)
+
+- Improved :command:`shell` help formatting when using IPython 9+.
+ (:issue:`6915`, :issue:`6980`)
+
+Bug fixes
+~~~~~~~~~
+
+- Setting :setting:`FILES_STORE` or :setting:`IMAGES_STORE` to ``None`` now
+ correctly disables the respective pipeline.
+ (:issue:`6964`, :issue:`6969`)
+
+- :class:`~scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware` now
+ uses the URL set in the ```` tag as the base URL when redirecting to
+ a relative URL.
+ (:issue:`7042`, :issue:`7047`)
+
+- Passing ``None`` as a value of the :reqmeta:`download_slot` request meta
+ key is now handled in the same way as not setting this meta key at all.
+ (:issue:`7172`)
+
+- Fixed parsing of the first line of ``robots.txt`` files that have a BOM.
+ (:issue:`6195`, :issue:`7095`)
+
+Documentation
+~~~~~~~~~~~~~
+
+- Added :ref:`documentation ` about download
+ handlers, their API and built-in handlers.
+ (:issue:`4944`, :issue:`7164`)
+
+- Added a section about the `scrapy-spider-metadata`_ library to the
+ :ref:`spider argument docs `.
+ (:issue:`6676`, :issue:`6957`, :issue:`7116`)
+
+ .. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/
+
+- Improved :ref:`the docs ` about coroutine-based
+ and Deferred-based APIs.
+ (:issue:`6800`, :issue:`7146`)
+
+- Other documentation improvements and fixes.
+ (:issue:`7058`, :issue:`7076`, :issue:`7109`, :issue:`7195`, :issue:`7198`)
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+- Switched from ``twisted.trial`` to ``pytest-twisted`` and replaced
+ remaining ``unittest`` and ``twisted.trial`` features with ``pytest`` ones.
+ (:issue:`6658`, :issue:`6873`, :issue:`6884`, :issue:`6938`)
+
+- Enabled fancy ``pytest`` asserts.
+ (:issue:`6888`)
+
+- Added `Sphinx Lint`_ to the ``pre-commit`` configuration.
+ (:issue:`6920`)
+
+ .. _Sphinx Lint: https://github.com/sphinx-contrib/sphinx-lint
+
+- CI and test improvements and fixes.
+ (:issue:`6649`,
+ :issue:`6769`,
+ :issue:`6821`,
+ :issue:`6835`,
+ :issue:`6836`,
+ :issue:`6846`,
+ :issue:`6883`,
+ :issue:`6885`,
+ :issue:`6889`,
+ :issue:`6905`,
+ :issue:`6928`,
+ :issue:`6933`,
+ :issue:`6941`,
+ :issue:`6942`,
+ :issue:`6945`,
+ :issue:`6947`,
+ :issue:`6960`,
+ :issue:`6968`,
+ :issue:`6972`,
+ :issue:`6974`,
+ :issue:`6996`,
+ :issue:`7003`,
+ :issue:`7012`,
+ :issue:`7013`,
+ :issue:`7050`,
+ :issue:`7059`,
+ :issue:`7070`,
+ :issue:`7073`,
+ :issue:`7118`,
+ :issue:`7127`,
+ :issue:`7141`,
+ :issue:`7143`,
+ :issue:`7145`,
+ :issue:`7173`)
+
+- Code cleanups.
+ (:issue:`6803`,
+ :issue:`6838`,
+ :issue:`6849`,
+ :issue:`6875`,
+ :issue:`6876`,
+ :issue:`6892`,
+ :issue:`6930`,
+ :issue:`6949`,
+ :issue:`6970`,
+ :issue:`6977`,
+ :issue:`6986`,
+ :issue:`7008`,
+ :issue:`7177`)
+
+.. _release-2.13.4:
+
+Scrapy 2.13.4 (2025-11-17)
+--------------------------
+
+Security bug fixes
+~~~~~~~~~~~~~~~~~~
+
+- Improved protection against decompression bombs in
+ :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
+ for responses compressed using the ``br`` and ``deflate`` methods: if a
+ single compressed chunk would be larger than the response size limit (see
+ :setting:`DOWNLOAD_MAXSIZE`) when decompressed, decompression is no longer
+ carried out. This is especially important for the ``br`` (Brotli) method
+ that can provide a very high compression ratio. Please, see the
+ `CVE-2025-6176`_ and `GHSA-2qfp-q593-8484`_ security advisories for more
+ information.
+ (:issue:`7134`)
+
+ .. _CVE-2025-6176: https://nvd.nist.gov/vuln/detail/CVE-2025-6176
+ .. _GHSA-2qfp-q593-8484: https://github.com/advisories/GHSA-2qfp-q593-8484
+
+Modified requirements
+~~~~~~~~~~~~~~~~~~~~~
+
+- The minimum supported version of the optional ``brotli`` package is now
+ ``1.2.0``.
+ (:issue:`7134`)
+
+- The ``brotlicffi`` and ``brotlipy`` packages can no longer be used to
+ decompress Brotli-compressed responses. Please install the ``brotli``
+ package instead.
+ (:issue:`7134`)
+
+Other changes
+~~~~~~~~~~~~~
+
+- Restricted the maximum supported Twisted version to ``25.5.0``, as Scrapy
+ currently uses some private APIs changed in later Twisted versions.
+ (:issue:`7142`)
+
+- Stopped setting the ``COVERAGE_CORE`` environment variable in tests, it
+ didn't have an effect but caused the ``coverage`` module to produce a
+ warning or an error.
+ (:issue:`7137`)
+
+- Removed the documentation build dependency on the deprecated
+ ``sphinx-hoverxref`` module.
+ (:issue:`6786`, :issue:`6922`)
+
+.. _release-2.13.3:
+
+Scrapy 2.13.3 (2025-07-02)
+--------------------------
+
+- Changed the values for :setting:`DOWNLOAD_DELAY` (from ``0`` to ``1``) and
+ :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (from ``8`` to ``1``) in the
+ default project template.
+ (:issue:`6597`, :issue:`6918`, :issue:`6923`)
+
+- Improved :class:`scrapy.core.engine.ExecutionEngine` logic related to
+ initialization and exception handling, fixing several cases where the
+ spider would crash, hang or log an unhandled exception.
+ (:issue:`6783`, :issue:`6784`, :issue:`6900`, :issue:`6908`, :issue:`6910`,
+ :issue:`6911`)
+
+- Fixed a Windows issue with :ref:`feed exports ` using
+ :class:`scrapy.extensions.feedexport.FileFeedStorage` that caused the file
+ to be created on the wrong drive.
+ (:issue:`6894`, :issue:`6897`)
+
+- Allowed running tests with Twisted 25.5.0+ again. Pytest 8.4.1+ is now
+ required for running tests in non-pinned envs as support for the new
+ Twisted version was added in that version.
+ (:issue:`6893`)
+
+- Fixed running tests with lxml 6.0.0+.
+ (:issue:`6919`)
+
+- Added a deprecation notice for
+ ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware`` to :ref:`the Scrapy
+ 2.11.2 release notes `.
+ (:issue:`6926`)
+
+- Updated :ref:`contribution docs ` to refer to ruff_
+ instead of black_.
+ (:issue:`6903`)
+
+- Added ``.venv/`` and ``.vscode/`` to ``.gitignore``.
+ (:issue:`6901`, :issue:`6907`)
+
+
+.. _release-2.13.2:
+
+Scrapy 2.13.2 (2025-06-09)
+--------------------------
+
+- Fixed a bug introduced in Scrapy 2.13.0 that caused results of request
+ errbacks to be ignored when the errback was called because of a downloader
+ error.
+ (:issue:`6861`, :issue:`6863`)
+
+- Added a note about the behavior change of
+ :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to its docs and
+ to the "Backward-incompatible changes" section of :ref:`the Scrapy 2.13.0
+ release notes `.
+ (:issue:`6866`)
+
+- Improved the message in the exception raised by
+ :func:`scrapy.utils.test.get_reactor_settings` when there is no reactor
+ installed.
+ (:issue:`6866`)
+
+- Updated the :class:`scrapy.crawler.CrawlerRunner` examples in
+ :ref:`topics-practices` to install the reactor explicitly, to fix
+ reactor-related errors with Scrapy 2.13.0 and later.
+ (:issue:`6865`)
+
+- Fixed ``scrapy fetch`` not working with scrapy-poet_.
+ (:issue:`6872`)
+
+- Fixed an exception produced by :class:`scrapy.core.engine.ExecutionEngine`
+ when it's closed before being fully initialized.
+ (:issue:`6857`, :issue:`6867`)
+
+- Improved the README, updated the Scrapy logo in it.
+ (:issue:`6831`, :issue:`6833`, :issue:`6839`)
+
+- Restricted the Twisted version used in tests to below 25.5.0, as some tests
+ fail with 25.5.0.
+ (:issue:`6878`, :issue:`6882`)
+
+- Updated type hints for Twisted 25.5.0 changes.
+ (:issue:`6882`)
+
+- Removed the old artwork.
+ (:issue:`6874`)
+
+
.. _release-2.13.1:
Scrapy 2.13.1 (2025-05-28)
@@ -44,12 +1976,12 @@ Highlights:
- The asyncio reactor is now enabled by default
- Replaced ``start_requests()`` (sync) with :meth:`~scrapy.Spider.start`
- (async) and changed how it is iterated.
+ (async) and changed how it is iterated
- Added the :reqmeta:`allow_offsite` request meta key
-- :ref:`Spider middlewares that don't support asynchronous spider output
- ` are deprecated
+- Spider middlewares that don't support asynchronous spider output are
+ deprecated
- Added a base class for :ref:`universal spider middlewares
`
@@ -126,6 +2058,15 @@ Backward-incompatible changes
also enforced for start requests.
(:issue:`6777`)
+- Calling :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` without
+ an installed reactor now raises an exception instead of installing a
+ reactor. This shouldn't affect normal Scrapy use cases, but it may affect
+ 3rd-party test suites that use Scrapy internals such as
+ :class:`~scrapy.crawler.Crawler` and don't install a reactor explicitly. If
+ you are affected by this change, you most likely need to install the
+ reactor before running Scrapy code that expects it to be installed.
+ (:issue:`6732`, :issue:`6735`)
+
- The ``from_settings()`` method of
:class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`,
deprecated in Scrapy 2.12.0, is removed earlier than the usual deprecation
@@ -178,13 +2119,11 @@ Deprecations
``start_queue_cls`` parameter.
(:issue:`6752`)
-- :ref:`Spider middlewares that don't support asynchronous spider output
- ` are deprecated. The async iterable
- downgrading feature, needed for using such middlewares with asynchronous
- callbacks and with other spider middlewares that produce asynchronous
- iterables, is also deprecated. Please update all such middlewares to
- support asynchronous spider output.
- (:issue:`6664`)
+- Spider middlewares that don't support asynchronous spider output are
+ deprecated. The async iterable downgrading feature, needed for using such
+ middlewares with asynchronous callbacks and with other spider middlewares
+ that produce asynchronous iterables, is also deprecated. Please update all
+ such middlewares to support asynchronous spider output. (:issue:`6664`)
- Functions that were imported from :mod:`w3lib.url` and re-exported in
:mod:`scrapy.utils.url` are now deprecated, you should import them from
@@ -236,9 +2175,9 @@ Deprecations
- The following modules and functions used only in tests are deprecated:
- - the ``scrapy/utils/testproc`` module
+ - the ``scrapy.utils.testproc`` module
- - the ``scrapy/utils/testsite`` module
+ - the ``scrapy.utils.testsite`` module
- ``scrapy.utils.test.assert_gcs_environ()``
@@ -271,7 +2210,7 @@ Deprecations
(:issue:`6708`, :issue:`6714`)
- ``scrapy.utils.versions.scrapy_components_versions()`` is deprecated, use
- :func:`scrapy.utils.versions.get_versions()` instead.
+ :func:`scrapy.utils.versions.get_versions` instead.
(:issue:`6582`)
- ``BaseDupeFilter.log()`` is deprecated. It does nothing and shouldn't be
@@ -443,9 +2382,8 @@ Documentation
- Documented the setting values set in the default project template.
(:issue:`6762`, :issue:`6775`)
-- Improved the :ref:`docs ` about asynchronous
- iterable support in spider middlewares.
- (:issue:`6688`)
+- Improved the docs about asynchronous iterable support in spider
+ middlewares. (:issue:`6688`)
- Improved the :ref:`docs ` about using
:class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based
@@ -609,7 +2547,7 @@ Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- User-defined cookies for HTTPS requests will have the ``secure`` flag set
- to ``True`` unless it's set to ``False`` explictly. This is important when
+ to ``True`` unless it's set to ``False`` explicitly. This is important when
these cookies are reused in HTTP requests, e.g. after a redirect to an HTTP
URL.
(:issue:`6357`)
@@ -644,7 +2582,7 @@ Backward-incompatible changes
``crawler.settings`` instead. When they call ``__init__()`` of the base
class they should pass the ``crawler`` argument to it too.
- A ``from_settings()`` method shouldn't be defined. Class-specific
- initialization code should go into either an overriden ``from_crawler()``
+ initialization code should go into either an overridden ``from_crawler()``
method or into ``__init__()``.
- It's now possible to override ``from_crawler()`` and it's not necessary
to call ``MediaPipeline.from_crawler()`` in it if other recommendations
@@ -728,7 +2666,7 @@ Deprecation removals
(:issue:`6109`, :issue:`6116`)
- A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that
- does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface
+ does not implement the ``ISpiderLoader`` interface
will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at
run time. Non-compliant classes have been triggering a deprecation warning
since Scrapy 1.0.0.
@@ -1164,6 +3102,17 @@ Security bug fixes
.. _defusedxml: https://github.com/tiran/defusedxml
+Deprecations
+~~~~~~~~~~~~
+
+- ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware`` (a spider
+ middleware) is now deprecated and not enabled by default. The new
+ downloader middleware with the same functionality,
+ :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`, is enabled
+ instead.
+ (:issue:`2241`, :issue:`6358`)
+
+
Bug fixes
~~~~~~~~~
@@ -1717,7 +3666,7 @@ Bug fixes
(:issue:`5914`, :issue:`5917`)
- Fixed an error breaking user handling of send failures in
- :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`)
+ :meth:`scrapy.mail.MailSender.send`. (:issue:`1611`, :issue:`5880`)
Documentation
~~~~~~~~~~~~~
@@ -2136,7 +4085,7 @@ Bug fixes
that does not match the asyncio event loop actually installed
(:issue:`5529`).
-- Fixed :meth:`Headers.getlist `
+- Fixed :meth:`Headers.getlist() `
returning only the last header (:issue:`5515`, :issue:`5526`).
- Fixed :class:`LinkExtractor
@@ -2381,7 +4330,7 @@ Modified requirements
~~~~~~~~~~~~~~~~~~~~~
- The h2_ dependency is now optional, only needed to
- :ref:`enable HTTP/2 support `. (:issue:`5113`)
+ :ref:`enable HTTP/2 support `. (:issue:`5113`)
.. _h2: https://pypi.org/project/h2/
@@ -2763,7 +4712,7 @@ Highlights:
- Official Python 3.9 support
-- Experimental :ref:`HTTP/2 support `
+- Experimental :ref:`HTTP/2 support `
- New :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function
to retry requests from spider callbacks
@@ -2794,7 +4743,7 @@ Deprecations
New features
~~~~~~~~~~~~
-- Experimental :ref:`HTTP/2 support ` through a new download handler
+- Experimental :ref:`HTTP/2 support ` through a new download handler
that can be assigned to the ``https`` protocol in the
:setting:`DOWNLOAD_HANDLERS` setting.
(:issue:`1854`, :issue:`4769`, :issue:`5058`, :issue:`5059`, :issue:`5066`)
@@ -3207,7 +5156,7 @@ Bug fixes
* The system file mode creation mask no longer affects the permissions of
files generated using the :command:`startproject` command (:issue:`4722`)
-* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names
+* ``scrapy.utils.iterators.xmliter`` now supports namespaced node names
(:issue:`861`, :issue:`4746`)
* :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can
@@ -3768,7 +5717,7 @@ Highlights:
* :ref:`FTP support ` for media pipelines
* New :attr:`Response.certificate `
attribute
-* IPv6 support through :setting:`DNS_RESOLVER`
+* IPv6 support through ``DNS_RESOLVER``
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -3876,7 +5825,7 @@ New features
:class:`twisted.internet.ssl.Certificate` object for HTTPS responses
(:issue:`2726`, :issue:`4054`)
-* A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support
+* A new ``DNS_RESOLVER`` setting allows enabling IPv6 support
(:issue:`1031`, :issue:`4227`)
* A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring
@@ -3956,7 +5905,7 @@ New features
components already supported (:issue:`4126`)
* :class:`scrapy.utils.python.MutableChain.__iter__` now returns ``self``,
- `allowing it to be used as a sequence `_
+ allowing it to be used as a sequence.
(:issue:`4153`)
@@ -4120,7 +6069,7 @@ The following changes may impact custom priority queue classes:
* A new keyword parameter has been added: ``key``. It is a string
that is always an empty string for memory queues and indicates the
- :setting:`JOB_DIR` value for disk queues.
+ :setting:`JOBDIR` value for disk queues.
* The parameter for disk queues that contains data from the previous
crawl, ``startprios`` or ``slot_startprios``, is now passed as a
@@ -4385,6 +6334,8 @@ Highlights:
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. skip: start
+
* Python 3.4 is no longer supported, and some of the minimum requirements of
Scrapy have also changed:
@@ -4404,7 +6355,7 @@ Backward-incompatible changes
consistency with similar classes (:issue:`3929`, :issue:`3982`)
* If you are using a custom context factory
- (:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`), its ``__init__`` method must
+ (``DOWNLOADER_CLIENTCONTEXTFACTORY``), its ``__init__`` method must
accept two new parameters: ``tls_verbose_logging`` and ``tls_ciphers``
(:issue:`2111`, :issue:`3392`, :issue:`3442`, :issue:`3450`)
@@ -4425,6 +6376,8 @@ Backward-incompatible changes
(:issue:`3804`, :issue:`3819`, :issue:`3897`, :issue:`3976`, :issue:`3998`,
:issue:`4036`)
+.. skip: end
+
See also :ref:`1.8-deprecation-removals` below.
@@ -4637,8 +6590,8 @@ Backward-incompatible changes
``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly.
* :class:`~scrapy.crawler.Crawler`,
- :class:`CrawlerRunner.crawl ` and
- :class:`CrawlerRunner.create_crawler `
+ :meth:`CrawlerRunner.crawl ` and
+ :meth:`CrawlerRunner.create_crawler `
no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they
only accept a :class:`~scrapy.spiders.Spider` subclass now.
@@ -4670,7 +6623,7 @@ New features
``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
:ref:`enabled ` for a significant
scheduling improvement on crawls targeting multiple web domains, at the
- cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
+ cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`)
* A new :attr:`.Request.cb_kwargs` attribute
provides a cleaner way to pass keyword arguments to callback methods
@@ -5263,7 +7216,7 @@ Docs
- Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY``
setting. (:issue:`2756`)
- Update Contributing docs, document new support channels
- (:issue:`2762`, issue:`3038`)
+ (:issue:`2762`, :issue:`3038`)
- Include references to Scrapy subreddit in the docs
- Fix broken links; use ``https://`` for external links
(:issue:`2978`, :issue:`2982`, :issue:`2958`)
@@ -5858,7 +7811,7 @@ This 1.1 release brings a lot of interesting features and bug fixes:
selectors engine without needing to upgrade Scrapy.
- HTTPS downloader now does TLS protocol negotiation by default,
instead of forcing TLS 1.0. You can also set the SSL/TLS method
- using the new :setting:`DOWNLOADER_CLIENT_TLS_METHOD`.
+ using the new ``DOWNLOADER_CLIENT_TLS_METHOD`` setting.
- These bug fixes may require your attention:
@@ -5893,8 +7846,7 @@ Keep reading for more details on other improvements and bug fixes.
Beta Python 3 Support
~~~~~~~~~~~~~~~~~~~~~
-We have been `hard at work to make Scrapy run on Python 3
-`_. As a result, now
+We have been hard at work to make Scrapy run on Python 3. As a result, now
you can run spiders on Python 3.3, 3.4 and 3.5 (Twisted >= 15.5 required). Some
features are still missing (and some may never be ported).
@@ -5962,7 +7914,7 @@ Additional New Features and Enhancements
- Other refactoring, optimizations and cleanup (:issue:`1476`, :issue:`1481`,
:issue:`1477`, :issue:`1315`, :issue:`1290`, :issue:`1750`, :issue:`1881`).
-.. _`Code of Conduct`: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
+.. _Code of Conduct: https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md
Deprecations and Removals
@@ -6724,7 +8676,7 @@ Enhancements
- Make ``RFPDupeFilter`` class easily subclassable (:issue:`533`)
- Improve test coverage and forthcoming Python 3 support (:issue:`525`)
- Promote startup info on settings and middleware to INFO level (:issue:`520`)
-- Support partials in ``get_func_args`` util (:issue:`506`, issue:`504`)
+- Support partials in ``get_func_args`` util (:issue:`506`, :issue:`504`)
- Allow running individual tests via tox (:issue:`503`)
- Update extensions ignored by link extractors (:issue:`498`)
- Add middleware methods to get files/images/thumbs paths (:issue:`490`)
@@ -7150,7 +9102,7 @@ New features and settings
- In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`)
- Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
- ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
- - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP`
+ - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP``
- check the documentation for more details
- Added builtin caching DNS resolver (:rev:`2728`)
- Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)
@@ -7292,7 +9244,7 @@ API changes
- ``scrapy.core.manager.ScrapyManager`` class renamed to ``scrapy.crawler.Crawler``
- ``scrapy.core.manager.scrapymanager`` singleton moved to ``scrapy.project.crawler``
- Moved module: ``scrapy.contrib.spidermanager`` to ``scrapy.spidermanager``
-- Spider Manager singleton moved from ``scrapy.spider.spiders`` to the ``spiders` attribute of ``scrapy.project.crawler`` singleton.
+- Spider Manager singleton moved from ``scrapy.spider.spiders`` to the ``spiders`` attribute of ``scrapy.project.crawler`` singleton.
- moved Stats Collector classes: (#204)
- ``scrapy.stats.collector.StatsCollector`` to ``scrapy.statscol.StatsCollector``
- ``scrapy.stats.collector.SimpledbStatsCollector`` to ``scrapy.contrib.statscol.SimpledbStatsCollector``
diff --git a/docs/requirements.in b/docs/requirements.in
new file mode 100644
index 000000000..3783dd1dc
--- /dev/null
+++ b/docs/requirements.in
@@ -0,0 +1,8 @@
+h2
+pydantic
+scrapy-spider-metadata
+sphinx
+sphinx-notfound-page
+sphinx-rtd-theme
+sphinx-rtd-dark-mode
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 103fb08d6..0f5969401 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,5 +1,197 @@
-sphinx==8.1.3
-sphinx-hoverxref==1.4.2
-sphinx-notfound-page==1.0.4
-sphinx-rtd-theme==3.0.2
+# This file was autogenerated by uv via the following command:
+# uv pip compile -p 3.13 docs/requirements.in -o docs/requirements.txt
+alabaster==1.0.0
+ # via sphinx
+annotated-types==0.7.0
+ # via pydantic
+attrs==26.1.0
+ # via
+ # service-identity
+ # twisted
+automat==25.4.16
+ # via twisted
+babel==2.18.0
+ # via sphinx
+certifi==2026.2.25
+ # via requests
+cffi==2.0.0
+ # via cryptography
+charset-normalizer==3.4.6
+ # via requests
+constantly==23.10.4
+ # via twisted
+cryptography==46.0.6
+ # via
+ # pyopenssl
+ # scrapy
+ # service-identity
+cssselect==1.4.0
+ # via
+ # parsel
+ # scrapy
+defusedxml==0.7.1
+ # via scrapy
+docutils==0.22.4
+ # via
+ # sphinx
+ # sphinx-markdown-builder
+ # sphinx-rtd-theme
+filelock==3.25.2
+ # via tldextract
+h2==4.3.0
+ # via -r docs/requirements.in
+hpack==4.1.0
+ # via h2
+hyperframe==6.1.0
+ # via h2
+hyperlink==21.0.0
+ # via twisted
+idna==3.11
+ # via
+ # hyperlink
+ # requests
+ # tldextract
+imagesize==2.0.0
+ # via sphinx
+incremental==24.11.0
+ # via twisted
+itemadapter==0.13.1
+ # via
+ # itemloaders
+ # scrapy
+itemloaders==1.4.0
+ # via scrapy
+jinja2==3.1.6
+ # via sphinx
+jmespath==1.1.0
+ # via
+ # itemloaders
+ # parsel
+lxml==6.0.2
+ # via
+ # parsel
+ # scrapy
+markupsafe==3.0.3
+ # via jinja2
+packaging==26.0
+ # via
+ # incremental
+ # parsel
+ # scrapy
+ # scrapy-spider-metadata
+ # sphinx
+ # sphinx-scrapy
+parsel==1.11.0
+ # via
+ # itemloaders
+ # scrapy
+protego==0.6.0
+ # via scrapy
+pyasn1==0.6.3
+ # via
+ # pyasn1-modules
+ # service-identity
+pyasn1-modules==0.4.2
+ # via service-identity
+pycparser==3.0
+ # via cffi
+pydantic==2.12.5
+ # via
+ # -r docs/requirements.in
+ # scrapy-spider-metadata
+pydantic-core==2.41.5
+ # via pydantic
+pydispatcher==2.0.7
+ # via scrapy
+pygments==2.19.2
+ # via sphinx
+pyopenssl==26.0.0
+ # via scrapy
+queuelib==1.9.0
+ # via scrapy
+requests==2.33.0
+ # via
+ # requests-file
+ # sphinx
+ # tldextract
+requests-file==3.0.1
+ # via tldextract
+roman-numerals==4.1.0
+ # via sphinx
+scrapy==2.14.2
+ # via scrapy-spider-metadata
+scrapy-spider-metadata==0.2.0
+ # via -r docs/requirements.in
+service-identity==24.2.0
+ # via scrapy
+snowballstemmer==3.0.1
+ # via sphinx
+sphinx==9.1.0
+ # via
+ # -r docs/requirements.in
+ # sphinx-copybutton
+ # sphinx-last-updated-by-git
+ # sphinx-llms-txt
+ # sphinx-markdown-builder
+ # sphinx-notfound-page
+ # sphinx-rtd-theme
+ # sphinx-scrapy
+ # sphinxcontrib-jquery
+sphinx-copybutton==0.5.2
+ # via sphinx-scrapy
+sphinx-last-updated-by-git==0.3.8
+ # via sphinx-sitemap
+sphinx-llms-txt @ git+https://github.com/zytedata/sphinx-llms-txt.git@5e8866cb0cc249aa2017ad9050b3b83a7ca16f69
+ # via sphinx-scrapy
+sphinx-markdown-builder @ git+https://github.com/zytedata/sphinx-markdown-builder.git@cfe4c0bfd7b4542f7e6b65a58cdf9ec765829940
+ # via sphinx-scrapy
+sphinx-notfound-page==1.1.0
+ # via -r docs/requirements.in
sphinx-rtd-dark-mode==1.3.0
+ # via -r docs/requirements.in
+sphinx-rtd-theme==3.1.0
+ # via
+ # -r docs/requirements.in
+ # sphinx-rtd-dark-mode
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8
+ # via -r docs/requirements.in
+sphinx-sitemap==2.9.0
+ # via sphinx-scrapy
+sphinxcontrib-applehelp==2.0.0
+ # via sphinx
+sphinxcontrib-devhelp==2.0.0
+ # via sphinx
+sphinxcontrib-htmlhelp==2.1.0
+ # via sphinx
+sphinxcontrib-jquery==4.1
+ # via sphinx-rtd-theme
+sphinxcontrib-jsmath==1.0.1
+ # via sphinx
+sphinxcontrib-qthelp==2.0.0
+ # via sphinx
+sphinxcontrib-serializinghtml==2.0.0
+ # via sphinx
+tabulate==0.10.0
+ # via sphinx-markdown-builder
+tldextract==5.3.1
+ # via scrapy
+twisted==25.5.0
+ # via scrapy
+typing-extensions==4.15.0
+ # via
+ # pydantic
+ # pydantic-core
+ # twisted
+ # typing-inspection
+typing-inspection==0.4.2
+ # via pydantic
+urllib3==2.6.3
+ # via requests
+w3lib==2.4.1
+ # via
+ # parsel
+ # scrapy
+zope-interface==8.2
+ # via
+ # scrapy
+ # twisted
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst
index 815501e66..75f15b3ae 100644
--- a/docs/topics/addons.rst
+++ b/docs/topics/addons.rst
@@ -21,10 +21,14 @@ The ``ADDONS`` setting is a dict in which every key is an add-on class or its
import path and the value is its priority.
This is an example where two add-ons are enabled in a project's
-``settings.py``::
+``settings.py``:
+
+.. skip: next
+
+.. code-block:: python
ADDONS = {
- 'path.to.someaddon': 0,
+ "path.to.someaddon": 0,
SomeAddonClass: 1,
}
@@ -56,7 +60,9 @@ the following methods:
:type settings: :class:`~scrapy.settings.BaseSettings`
The settings set by the add-on should use the ``addon`` priority (see
-:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`)::
+:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):
+
+.. code-block:: python
class MyAddon:
def update_settings(self, settings):
@@ -88,7 +94,7 @@ recommend that such custom components should be written in the following way:
1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the
default Scrapy one (e.g.
- ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead
+ ``scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler``), but instead
be able to load the class of the fallback component from a special setting
(e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use
it.
@@ -98,9 +104,9 @@ recommend that such custom components should be written in the following way:
(``MY_FALLBACK_DOWNLOAD_HANDLER`` mentioned earlier) and set the default
setting to the component provided by the add-on (e.g.
``MyDownloadHandler``). If the fallback setting is already set by the user,
- they shouldn't change it.
+ it should not be changed.
3. This way, if there are several add-ons that want to modify the same setting,
- all of them will fallback to the component from the previous one and then to
+ all of them will fall back to the component from the previous one and then to
the Scrapy default. The order of that depends on the priority order in the
``ADDONS`` setting.
@@ -166,9 +172,7 @@ Use a fallback component:
.. code-block:: python
- from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
- from scrapy.utils.misc import build_from_crawler
-
+ from scrapy.utils.misc import build_from_crawler, load_object
FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
@@ -176,16 +180,19 @@ Use a fallback component:
class MyHandler:
lazy = False
- def __init__(self, settings, crawler):
- dhcls = load_object(settings.get(FALLBACK_SETTING))
+ def __init__(self, crawler):
+ dhcls = load_object(crawler.settings.get(FALLBACK_SETTING))
self._fallback_handler = build_from_crawler(dhcls, crawler)
- def download_request(self, request, spider):
+ async def download_request(self, request):
if request.meta.get("my_params"):
# handle the request
...
else:
- return self._fallback_handler.download_request(request, spider)
+ return await self._fallback_handler.download_request(request)
+
+ async def close(self):
+ pass
class MyAddon:
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index d90eb0bad..7ff8d3464 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -172,46 +172,15 @@ SpiderLoader API
.. module:: scrapy.spiderloader
:synopsis: The spider loader
-.. class:: SpiderLoader
+Custom spider loaders can be employed by specifying their path in the
+:setting:`SPIDER_LOADER_CLASS` project setting. They must implement
+:class:`SpiderLoaderProtocol`.
- This class is in charge of retrieving and handling the spider classes
- defined across the project.
+.. autoclass:: SpiderLoaderProtocol
+ :members:
- Custom spider loaders can be employed by specifying their path in the
- :setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement
- the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an
- errorless execution.
-
- .. method:: from_settings(settings)
-
- This class method is used by Scrapy to create an instance of the class.
- It's called with the current project settings, and it loads the spiders
- found recursively in the modules of the :setting:`SPIDER_MODULES`
- setting.
-
- :param settings: project settings
- :type settings: :class:`~scrapy.settings.Settings` instance
-
- .. method:: load(spider_name)
-
- Get the Spider class with the given name. It'll look into the previously
- loaded spiders for a spider class with name ``spider_name`` and will raise
- a KeyError if not found.
-
- :param spider_name: spider class name
- :type spider_name: str
-
- .. method:: list()
-
- Get the names of the available spiders in the project.
-
- .. method:: find_by_request(request)
-
- List the spiders' names that can handle the given request. Will try to
- match the request's url against the domains of the spiders.
-
- :param request: queried request
- :type request: :class:`~scrapy.Request` instance
+.. autoclass:: SpiderLoader
+ :members:
.. autoclass:: DummySpiderLoader
@@ -280,13 +249,13 @@ class (which they all inherit from).
The following methods are not part of the stats collection api but instead
used when implementing custom stats collectors:
- .. method:: open_spider(spider)
+ .. method:: open_spider()
- Open the given spider for stats collection.
+ Open the spider for stats collection.
- .. method:: close_spider(spider)
+ .. method:: close_spider()
- Close the given spider. After this is called, no more specific stats
+ Close the spider. After this is called, no more specific stats
can be accessed or collected.
Engine API
diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst
index e8c510ea5..c60c43f3c 100644
--- a/docs/topics/architecture.rst
+++ b/docs/topics/architecture.rst
@@ -63,7 +63,7 @@ this:
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`).
8. The :ref:`Engine ` sends processed items to
- :ref:`Item Pipelines `, then send processed Requests to
+ :ref:`Item Pipelines `, then sends processed Requests to
the :ref:`Scheduler ` and asks for possible next Requests
to crawl.
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index ad5c71fbf..afccb491d 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -4,21 +4,27 @@
asyncio
=======
-.. versionadded:: 2.0
+Scrapy supports :mod:`asyncio` natively. New projects created with
+:command:`startproject` have asyncio enabled by default, and you can use
+:mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine
+`.
-Scrapy has partial support for :mod:`asyncio`. After you :ref:`install the
-asyncio reactor `, you may use :mod:`asyncio` and
-:mod:`asyncio`-powered libraries in any :doc:`coroutine `.
+The rest of this page covers advanced topics. If you are starting a new project,
+no additional setup is needed.
.. _install-asyncio:
-Installing the asyncio reactor
-==============================
+Configuring the asyncio reactor
+===============================
-To enable :mod:`asyncio` support, your :setting:`TWISTED_REACTOR` setting needs
-to be set to ``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``,
-which is the default value.
+New projects generated with :command:`startproject` have the asyncio
+reactor configured by default. No manual setup is needed.
+
+The :setting:`TWISTED_REACTOR` setting controls which Twisted reactor Scrapy
+uses. Its default value is
+``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``, which enables
+:mod:`asyncio` support.
If you are using :class:`~scrapy.crawler.AsyncCrawlerRunner` or
:class:`~scrapy.crawler.CrawlerRunner`, you also need to
@@ -49,6 +55,7 @@ You can usually fix the issue by moving those offending module-level Twisted
imports to the method or function definitions where they are used. For example,
if you have something like:
+.. skip: next
.. code-block:: python
from twisted.internet import reactor
@@ -99,6 +106,10 @@ Scrapy API requires passing a Deferred to it) using the following helpers:
.. autofunction:: scrapy.utils.defer.deferred_from_coro
.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f
+The following function helps with a reverse wrapping:
+
+.. autofunction:: scrapy.utils.defer.ensure_awaitable
+
.. _enforce-asyncio-requirement:
@@ -129,6 +140,181 @@ example:
.. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed
+.. _asyncio-without-reactor:
+
+Using Scrapy without a Twisted reactor
+======================================
+
+.. versionadded:: 2.15.0
+
+.. warning::
+ This is currently experimental and may not be suitable for production use.
+
+.. note:: As the Twisted download handlers cannot be used without a reactor,
+ the default download handler in this mode is
+ :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. You
+ will need to additionally install the :ref:`httpx ` extra to use
+ it, unless you switch to some different handler.
+
+It's possible to use Scrapy without installing a Twisted reactor at all, by
+setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this
+mode Scrapy will use the asyncio event loop directly, and most of the Scrapy
+functionality will work in the same way.
+
+Doing this provides several benefits in certain use cases:
+
+* A Twisted reactor, once stopped, cannot be started again. This prevents, for
+ example, using several instances of
+ :class:`~scrapy.crawler.AsyncCrawlerProcess` in the same process when they
+ use a reactor, but with ``TWISTED_REACTOR_ENABLED=False`` it becomes
+ possible.
+* There may be limitations imposed by
+ :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` and related
+ Twisted code, such as the requirement of using
+ :class:`~asyncio.SelectorEventLoop` on Windows (see :ref:`asyncio-windows`),
+ that do not apply if the reactor is not used.
+* :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` manages the
+ underlying event loop, and while :class:`~scrapy.crawler.AsyncCrawlerRunner`
+ can use a pre-existing reactor which, in turn, can use a pre-existing event
+ loop, it's easier to use :class:`~scrapy.crawler.AsyncCrawlerRunner` with a
+ pre-existing loop directly.
+* Omitting the reactor machinery may improve performance and reliability.
+
+Limitations
+-----------
+
+As some Scrapy features and components require a reactor, they don't work and
+are disabled without it. Replacements that don't require a reactor may be added
+in future Scrapy versions. The following features are not available:
+
+* The default HTTP(S) download handler,
+ :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` (this
+ is likely the biggest difference; Scrapy provides an HTTP(S) download handler
+ that doesn't require a reactor and will be used instead of it:
+ :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`)
+* :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler`
+* :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`
+* :ref:`topics-telnetconsole`
+* :class:`~scrapy.crawler.CrawlerRunner` and
+ :class:`~scrapy.crawler.CrawlerProcess`
+ (:class:`~scrapy.crawler.AsyncCrawlerProcess` and
+ :class:`~scrapy.crawler.AsyncCrawlerRunner` are available)
+* Twisted-specific DNS resolvers (the :setting:`TWISTED_DNS_RESOLVER` setting)
+* User and 3rd-party code that requires a reactor (see :ref:`below
+ ` for examples)
+
+Note that importing Twisted modules and, among other things, creating and using
+:class:`~twisted.internet.defer.Deferred` objects doesn't require a reactor, so
+code that uses :class:`~twisted.internet.defer.Deferred`,
+:class:`~twisted.python.failure.Failure` and some other Twisted APIs will not
+necessarily stop working.
+
+Other differences
+-----------------
+
+When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``, Scrapy will change
+the defaults of some other settings:
+
+* :setting:`TELNETCONSOLE_ENABLED` is set to ``False``.
+* The ``"http"`` and ``"https"`` keys in :setting:`DOWNLOAD_HANDLERS_BASE` are
+ set to ``"scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler"``.
+* The ``"ftp"`` key in :setting:`DOWNLOAD_HANDLERS_BASE` is set to ``None``.
+
+Thus, :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` is
+used by default for making HTTP(S) requests. Please refer to its documentation
+for its differences and limitations compared to
+:class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`.
+
+Additionally, :class:`~scrapy.crawler.AsyncCrawlerProcess` will install a
+:term:`meta path finder` that prevents :mod:`twisted.internet.reactor` from
+being imported. It will be uninstalled when :meth:`AsyncCrawlerProcess.start()
+` exits.
+
+.. _asyncio-without-reactor-migrate:
+
+Adding support to existing code
+-------------------------------
+
+Code that doesn't directly use Twisted APIs or APIs that depend on Twisted ones
+doesn't need special support for running without a reactor.
+
+Here are some examples of APIs and patterns that need a replacement:
+
+* Using :meth:`reactor.callLater()
+ ` for sleeping or delayed calls.
+ You can use :meth:`asyncio.loop.call_later` instead.
+* Using :func:`twisted.internet.threads.deferToThread`,
+ :meth:`reactor.callFromThread()
+ ` and related APIs to
+ execute code in other threads. You can use :func:`asyncio.to_thread`,
+ :meth:`asyncio.loop.call_soon_threadsafe` and related APIs instead.
+* Using :class:`twisted.internet.task.LoopingCall` for scheduling repeated
+ tasks. As there is no direct replacement in the standard library, you may
+ need to write your own one using :func:`asyncio.sleep` in a task.
+* Using Twisted network client and server APIs (:meth:`reactor.connectTCP()
+ `,
+ :meth:`reactor.listenTCP()
+ `,
+ :mod:`twisted.web.client`, :mod:`twisted.mail.smtp` etc.). You can use other
+ built-in or 3rd-party libraries for this.
+* Using :class:`~scrapy.crawler.CrawlerProcess` or
+ :class:`~scrapy.crawler.CrawlerRunner`. You should use
+ :class:`~scrapy.crawler.AsyncCrawlerProcess` or
+ :class:`~scrapy.crawler.AsyncCrawlerRunner` respectively instead.
+* Checking whether ``asyncio`` support is available with
+ :func:`scrapy.utils.reactor.is_asyncio_reactor_installed`. You should use
+ :func:`scrapy.utils.asyncio.is_asyncio_available` instead.
+
+Scrapy provides unified helpers for some of these examples:
+
+.. autofunction:: scrapy.utils.asyncio.sleep
+.. autofunction:: scrapy.utils.asyncio.call_later
+.. autofunction:: scrapy.utils.asyncio.create_looping_call
+.. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall
+.. autofunction:: scrapy.utils.asyncio.run_in_thread
+
+If your code needs to know whether the reactor is available, you can either
+check for the value of the :setting:`TWISTED_REACTOR_ENABLED` setting (you need
+access to the :class:`~scrapy.crawler.Crawler` instance to do this) or use the
+following function:
+
+.. autofunction:: scrapy.utils.reactorless.is_reactorless
+
+In general, code that doesn't use the reactor (directly or indirectly) can be
+used unmodified both with the asyncio reactor and without a reactor. This
+includes code that converts Deferreds to futures and vice versa as described in
+:ref:`asyncio-await-dfd`.
+
+Troubleshooting
+---------------
+
+**ImportError: Import of twisted.internet.reactor is forbidden when running
+without a Twisted reactor [...]:** Scrapy is configured to run without a
+reactor, but some code imported :mod:`twisted.internet.reactor`, most likely
+because that code needs a reactor to be used. You need to stop using this code
+or set :setting:`TWISTED_REACTOR_ENABLED` back to ``True``. It's also possible
+that the reactor isn't really needed but was installed due to the problem
+described in :ref:`asyncio-preinstalled-reactor`, in which case it should be
+enough to fix the problematic imports.
+
+**RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is
+installed:** Scrapy is configured to run without a reactor, but a reactor is
+already installed before the Scrapy code is executed. If you are trying to set
+:setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings
+`, it's currently unsupported.
+
+**RuntimeError: We expected a Twisted reactor to be installed but it isn't:**
+Scrapy is configured to run with a reactor and not to install one, but a
+reactor wasn't installed before the Scrapy code is executed. If you are trying
+to set :setting:`TWISTED_REACTOR_ENABLED` via :ref:`per-spider settings
+`, it's currently unsupported.
+
+**RuntimeError: doesn't support TWISTED_REACTOR_ENABLED=False:** The
+listed class cannot be used with :setting:`TWISTED_REACTOR_ENABLED` set to
+``False``. There may be a replacement in the :ref:`documentation above
+` or the documentation of the affected class.
+
+
.. _asyncio-windows:
Windows-specific notes
@@ -140,8 +326,7 @@ implementations, :class:`~asyncio.ProactorEventLoop` (default) and
:class:`~asyncio.SelectorEventLoop` works with Twisted.
Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop`
-automatically when you change the :setting:`TWISTED_REACTOR` setting or call
-:func:`~scrapy.utils.reactor.install_reactor`.
+automatically when installing the asyncio reactor.
.. note:: Other libraries you use may require
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
@@ -149,6 +334,9 @@ automatically when you change the :setting:`TWISTED_REACTOR` setting or call
them together with Scrapy on Windows (but you should be able to use
them on WSL or native Linux).
+.. note:: This problem doesn't apply when not using the reactor, see
+ :ref:`asyncio-without-reactor`.
+
.. _playwright: https://github.com/microsoft/playwright-python
diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index 5bd72fa15..33289545c 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -37,8 +37,7 @@ processed in parallel.
Instead of adjusting the delays one can just set a small fixed
download delay and impose hard limits on concurrency using
-:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
-:setting:`CONCURRENT_REQUESTS_PER_IP` options. It will provide a similar
+:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. It will provide a similar
effect, but there are some important differences:
* because the download delay is small there will be occasional bursts
@@ -71,13 +70,12 @@ AutoThrottle algorithm adjusts download delays based on the following rules:
.. note:: The AutoThrottle extension honours the standard Scrapy settings for
concurrency and delay. This means that it will respect
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
- :setting:`CONCURRENT_REQUESTS_PER_IP` options and
never set a download delay lower than :setting:`DOWNLOAD_DELAY`.
.. _download-latency:
In Scrapy, the download latency is measured as the time elapsed between
-establishing the TCP connection and receiving the HTTP headers.
+sending the request and receiving the HTTP headers.
Note that these latencies are very hard to measure accurately in a cooperative
multitasking environment because Scrapy may be busy processing a spider
@@ -90,6 +88,8 @@ server) is, and this extension builds on that premise.
Prevent specific requests from triggering slot delay adjustments
================================================================
+.. versionadded:: 2.12.0
+
AutoThrottle adjusts the delay of download slots based on the latencies of
responses that belong to that download slot. The only exceptions are non-200
responses, which are only taken into account to increase that delay, but
@@ -106,10 +106,9 @@ delay of its download slot:
Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True})
Note, however, that AutoThrottle still determines the starting delay of every
-download slot by setting the ``download_delay`` attribute on the running
-spider. If you want AutoThrottle not to impact a download slot at all, in
-addition to setting this meta key in all requests that use that download slot,
-you might want to set a custom value for the ``delay`` attribute of that
+download slot. If you want AutoThrottle not to impact a download slot at all,
+in addition to setting this meta key in all requests that use that download
+slot, you might want to set a custom value for the ``delay`` attribute of that
download slot, e.g. using :setting:`DOWNLOAD_SLOTS`.
Settings
@@ -123,7 +122,6 @@ The settings used to control the AutoThrottle extension are:
* :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`
* :setting:`AUTOTHROTTLE_DEBUG`
* :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
-* :setting:`CONCURRENT_REQUESTS_PER_IP`
* :setting:`DOWNLOAD_DELAY`
For more information see :ref:`autothrottle-algorithm`.
@@ -171,12 +169,10 @@ a higher value (e.g. ``2.0``) to increase the throughput and the load on remote
servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value
(e.g. ``0.5``) makes the crawler more conservative and polite.
-Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
-and :setting:`CONCURRENT_REQUESTS_PER_IP` options are still respected
+Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is still respected
when AutoThrottle extension is enabled. This means that if
``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than
-:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
-:setting:`CONCURRENT_REQUESTS_PER_IP`, the crawler won't reach this number
+:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, the crawler won't reach this number
of concurrent requests.
At every given time point Scrapy can be sending more or less concurrent
diff --git a/docs/topics/benchmarking.rst b/docs/topics/benchmarking.rst
index b704e54ed..e8ddec00c 100644
--- a/docs/topics/benchmarking.rst
+++ b/docs/topics/benchmarking.rst
@@ -83,4 +83,4 @@ and how well it's written.
Use scrapy-bench_ for more complex benchmarking.
-.. _scrapy-bench: https://github.com/scrapy/scrapy-bench
\ No newline at end of file
+.. _scrapy-bench: https://github.com/scrapy/scrapy-bench
diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index 248e38b61..cace1f883 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -41,19 +41,6 @@ efficient broad crawl.
.. _broad-crawls-scheduler-priority-queue:
-Use the right :setting:`SCHEDULER_PRIORITY_QUEUE`
-=================================================
-
-Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.
-It works best during single-domain crawl. It does not work well with crawling
-many different domains in parallel
-
-To apply the recommended priority queue use:
-
-.. code-block:: python
-
- SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue"
-
.. _broad-crawls-concurrency:
Increase concurrency
@@ -61,12 +48,7 @@ Increase concurrency
Concurrency is the number of requests that are processed in parallel. There is
a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that
-can be set either per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) or per
-IP (:setting:`CONCURRENT_REQUESTS_PER_IP`).
-
-.. note:: The scheduler priority queue :ref:`recommended for broad crawls
- ` does not support
- :setting:`CONCURRENT_REQUESTS_PER_IP`.
+can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`).
The default global concurrency limit in Scrapy is not suitable for crawling
many different domains in parallel, so you will want to increase it. How much
@@ -143,7 +125,7 @@ To disable cookies use:
Disable retries
===============
-Retrying failed HTTP requests can slow down the crawls substantially, specially
+Retrying failed HTTP requests can slow down the crawls substantially, especially
when sites causes are very slow (or fail) to respond, thus causing a timeout
error which gets retried many times, unnecessarily, preventing crawler capacity
to be reused for other domains.
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 6ffb8ae93..343193627 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -114,8 +114,8 @@ some usage help and the available commands::
scrapy [options] [args]
Available commands:
- crawl Run a spider
fetch Fetch a URL using the Scrapy downloader
+ runspider Run a spider from a Python file, no project required
[...]
The first line will print the currently active project if you're inside a
@@ -163,8 +163,8 @@ information on which commands must be run from inside projects, and which not.
Also keep in mind that some commands may have slightly different behaviours
when running them from inside projects. For example, the fetch command will use
-spider-overridden behaviours (such as the ``user_agent`` attribute to override
-the user-agent) if the url being fetched is associated with some specific
+spider-overridden behaviours (such as the ``custom_settings`` attribute to
+override settings) if the url being fetched is associated with some specific
spider. This is intentional, as the ``fetch`` command is meant to be used to
check how spiders are downloading pages.
@@ -199,6 +199,7 @@ Global commands:
* :command:`fetch`
* :command:`view`
* :command:`version`
+* :command:`bench`
Project-only commands:
@@ -207,7 +208,6 @@ Project-only commands:
* :command:`list`
* :command:`edit`
* :command:`parse`
-* :command:`bench`
.. command:: startproject
@@ -233,9 +233,6 @@ genspider
* Syntax: ``scrapy genspider [-t template] ``
* Requires project: *no*
-.. versionadded:: 2.6.0
- The ability to pass a URL instead of a domain.
-
Creates a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ```` parameter is set as the spider's ``name``, while ```` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
Usage example::
@@ -266,7 +263,9 @@ crawl
* Syntax: ``scrapy crawl ``
* Requires project: *yes*
-Start crawling using a spider.
+Start crawling using the spider with the given :attr:`~scrapy.Spider.name`,
+which must be one of those that :command:`list` reports. To run a spider from a
+file instead, use :command:`runspider`.
Supported options:
@@ -312,11 +311,25 @@ Usage examples::
* parse_item
$ scrapy check
- [FAILED] first_spider:parse_item
- >>> 'RetailPricex' field is missing
+ F.F.
+ ======================================================================
+ FAIL: [first_spider] parse (@returns post-hook)
+ ----------------------------------------------------------------------
+ Traceback (most recent call last):
+ ...
+ scrapy.exceptions.ContractFail: Returned 92 requests, expected 0..4
- [FAILED] first_spider:parse
- >>> Returned 92 requests, expected 0..4
+ ======================================================================
+ FAIL: [first_spider] parse_item (@scrapes post-hook)
+ ----------------------------------------------------------------------
+ Traceback (most recent call last):
+ ...
+ scrapy.exceptions.ContractFail: Missing fields: RetailPricex
+
+ ----------------------------------------------------------------------
+ Ran 4 contracts in 0.174s
+
+ FAILED (failures=2)
.. skip: end
@@ -380,7 +393,7 @@ Supported options:
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
-* ``--headers``: print the response's HTTP headers instead of the response's body
+* ``--headers``: print the request's and response's HTTP headers instead of the response's body
* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
@@ -390,15 +403,19 @@ Usage examples::
[ ... html content here ... ]
$ scrapy fetch --nolog --headers http://www.example.com/
- {'Accept-Ranges': ['bytes'],
- 'Age': ['1263 '],
- 'Connection': ['close '],
- 'Content-Length': ['596'],
- 'Content-Type': ['text/html; charset=UTF-8'],
- 'Date': ['Wed, 18 Aug 2010 23:59:46 GMT'],
- 'Etag': ['"573c1-254-48c9c87349680"'],
- 'Last-Modified': ['Fri, 30 Jul 2010 15:30:18 GMT'],
- 'Server': ['Apache/2.2.3 (CentOS)']}
+ > Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
+ > Accept-Language: en
+ > User-Agent: Scrapy/2.16.0 (+https://scrapy.org)
+ > Accept-Encoding: gzip, deflate, br
+ >
+ < Date: Wed, 08 Jul 2026 06:15:01 GMT
+ < Content-Type: text/html
+ < Server: cloudflare
+ < Last-Modified: Wed, 01 Jul 2026 17:50:18 GMT
+ < Allow: GET, HEAD
+ < Cf-Cache-Status: HIT
+ < Age: 8184
+ < Cf-Ray: a17cf3b80eddf141-DME
.. command:: view
@@ -479,7 +496,7 @@ Supported options:
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
-* ``--a NAME=VALUE``: set spider argument (may be repeated)
+* ``-a NAME=VALUE``: set spider argument (may be repeated)
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
response
@@ -509,8 +526,6 @@ Supported options:
* ``--output`` or ``-o``: dump scraped items to a file
- .. versionadded:: 2.3
-
.. skip: start
Usage example::
@@ -558,8 +573,9 @@ runspider
* Syntax: ``scrapy runspider ``
* Requires project: *no*
-Run a spider self-contained in a Python file, without having to create a
-project.
+Run the spider defined in the given Python file, without requiring a project.
+
+Supported options: the same as :command:`crawl`.
Example usage::
@@ -587,6 +603,47 @@ bench
Run a quick benchmark test. :ref:`benchmarking`.
+.. _topics-commands-crawlerprocess:
+
+Commands that run a crawl
+=========================
+
+Many commands need to run a crawl of some kind, running either a user-provided
+spider or a special internal one:
+
+* :command:`bench`
+* :command:`check`
+* :command:`crawl`
+* :command:`fetch`
+* :command:`parse`
+* :command:`runspider`
+* :command:`shell`
+* :command:`view`
+
+They use an internal instance of :class:`scrapy.crawler.AsyncCrawlerProcess` or
+:class:`scrapy.crawler.CrawlerProcess` for this. In most cases this detail
+shouldn't matter to the user running the command, but when the user :ref:`needs
+a non-default Twisted reactor `, it may be important.
+
+Scrapy decides which of these two classes to use based on the value of the
+:setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED` settings.
+With :setting:`TWISTED_REACTOR_ENABLED` set to ``False`` it will use
+:class:`~scrapy.crawler.AsyncCrawlerProcess`. Otherwise, if the
+:setting:`TWISTED_REACTOR` value is the default one
+(``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``),
+:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used, otherwise
+:class:`~scrapy.crawler.CrawlerProcess` will be used. The :ref:`spider settings
+` are not taken into account when doing this, as they are
+loaded after this decision is made. This may cause an error if the
+project-level setting is set to :ref:`the asyncio reactor `
+(:ref:`explicitly ` or :ref:`by using the Scrapy default
+`) and :ref:`the setting of the spider being run
+` is set to :ref:`a different one `, because
+:class:`~scrapy.crawler.AsyncCrawlerProcess` only supports the asyncio reactor.
+In this case you should set the :setting:`FORCE_CRAWLER_PROCESS` setting to
+``True`` (at the project level or via the command line) so that Scrapy uses
+:class:`~scrapy.crawler.CrawlerProcess` which supports all reactors.
+
Custom project commands
=======================
@@ -611,6 +668,8 @@ Example:
COMMANDS_MODULE = "mybot.commands"
+.. note:: This is a :ref:`pre-crawler setting `.
+
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
Register commands via setup.py entry points
diff --git a/docs/topics/components.rst b/docs/topics/components.rst
index 56f8c6498..354375577 100644
--- a/docs/topics/components.rst
+++ b/docs/topics/components.rst
@@ -9,39 +9,22 @@ A Scrapy component is any class whose objects are built using
That includes the classes that you may assign to the following settings:
-- :setting:`ADDONS`
-
-- :setting:`DNS_RESOLVER`
-
-- :setting:`DOWNLOAD_HANDLERS`
-
-- :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`
-
-- :setting:`DOWNLOADER_MIDDLEWARES`
-
-- :setting:`DUPEFILTER_CLASS`
-
-- :setting:`EXTENSIONS`
-
-- :setting:`FEED_EXPORTERS`
-
-- :setting:`FEED_STORAGES`
-
-- :setting:`ITEM_PIPELINES`
-
-- :setting:`SCHEDULER`
-
-- :setting:`SCHEDULER_DISK_QUEUE`
-
-- :setting:`SCHEDULER_MEMORY_QUEUE`
-
-- :setting:`SCHEDULER_PRIORITY_QUEUE`
-
-- :setting:`SCHEDULER_START_DISK_QUEUE`
-
-- :setting:`SCHEDULER_START_MEMORY_QUEUE`
-
-- :setting:`SPIDER_MIDDLEWARES`
+- :setting:`ADDONS`
+- :setting:`DOWNLOAD_HANDLERS`
+- :setting:`DOWNLOADER_MIDDLEWARES`
+- :setting:`DUPEFILTER_CLASS`
+- :setting:`EXTENSIONS`
+- :setting:`FEED_EXPORTERS`
+- :setting:`FEED_STORAGES`
+- :setting:`ITEM_PIPELINES`
+- :setting:`SCHEDULER`
+- :setting:`SCHEDULER_DISK_QUEUE`
+- :setting:`SCHEDULER_MEMORY_QUEUE`
+- :setting:`SCHEDULER_PRIORITY_QUEUE`
+- :setting:`SCHEDULER_START_DISK_QUEUE`
+- :setting:`SCHEDULER_START_MEMORY_QUEUE`
+- :setting:`SPIDER_MIDDLEWARES`
+- :setting:`TWISTED_DNS_RESOLVER`
Third-party Scrapy components may also let you define additional Scrapy
components, usually configurable through :ref:`settings `, to
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 61aef4bbb..df67bee02 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -30,43 +30,15 @@ You can use the following contracts:
.. module:: scrapy.contracts.default
-.. class:: UrlContract
+.. autoclass:: UrlContract
- This contract (``@url``) sets the sample URL used when checking other
- contract conditions for this spider. This contract is mandatory. All
- callbacks lacking this contract are ignored when running the checks::
+.. autoclass:: CallbackKeywordArgumentsContract
- @url url
+.. autoclass:: MetadataContract
-.. class:: CallbackKeywordArgumentsContract
+.. autoclass:: ReturnsContract
- This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs `
- attribute for the sample request. It must be a valid JSON dictionary.
- ::
-
- @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
-
-.. class:: MetadataContract
-
- This contract (``@meta``) sets the :attr:`meta `
- attribute for the sample request. It must be a valid JSON dictionary.
- ::
-
- @meta {"arg1": "value1", "arg2": "value2", ...}
-
-.. class:: ReturnsContract
-
- This contract (``@returns``) sets lower and upper bounds for the items and
- requests returned by the spider. The upper bound is optional::
-
- @returns item(s)|request(s) [min [max]]
-
-.. class:: ScrapesContract
-
- This contract (``@scrapes``) checks that all the items returned by the
- callback have the specified fields::
-
- @scrapes field_1 field_2 ...
+.. autoclass:: ScrapesContract
Use the :command:`check` command to run the contract checks.
@@ -89,30 +61,16 @@ override three methods:
.. module:: scrapy.contracts
-.. class:: Contract(method, *args)
+.. autoclass:: Contract
- :param method: callback function to which the contract is associated
- :type method: collections.abc.Callable
+ .. automethod:: adjust_request_args
- :param args: list of arguments passed into the docstring (whitespace
- separated)
- :type args: list
-
- .. method:: Contract.adjust_request_args(args)
-
- This receives a ``dict`` as an argument containing default arguments
- for request object. :class:`~scrapy.Request` is used by default,
- but this can be changed with the ``request_cls`` attribute.
- If multiple contracts in chain have this attribute defined, the last one is used.
-
- Must return the same or a modified version of it.
-
- .. method:: Contract.pre_process(response)
+ .. method:: pre_process(response)
This allows hooking in various checks on the response received from the
sample request, before it's being passed to the callback.
- .. method:: Contract.post_process(output)
+ .. method:: post_process(output)
This allows processing the output of the callback. Iterators are
converted to lists before being passed to this hook.
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 2c0df5e0f..b7ddb0a57 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -4,8 +4,6 @@
Coroutines
==========
-.. versionadded:: 2.0
-
Scrapy :ref:`supports ` the :ref:`coroutine syntax `
(i.e. ``async def``).
@@ -18,19 +16,14 @@ Supported callables
The following callables may be defined as coroutines using ``async def``, and
hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
-- The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be
+- The :meth:`~scrapy.Spider.start` spider method, which *must* be
defined as an :term:`asynchronous generator`.
.. versionadded:: 2.13
-- :class:`~scrapy.Request` callbacks.
-
- If you are using any custom or third-party :ref:`spider middleware
- `, see :ref:`sync-async-spider-middleware`.
-
- .. versionchanged:: 2.7
- Output of async callbacks is now processed asynchronously instead of
- collecting all of it first.
+- :class:`~scrapy.Request` :ref:`callbacks `, which may
+ also be defined as :term:`asynchronous generators `.
- The :meth:`process_item` method of
:ref:`item pipelines `.
@@ -45,15 +38,9 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- The
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
- method of :ref:`spider middlewares `.
-
- If defined as a coroutine, it must be an :term:`asynchronous generator`.
- The input ``result`` parameter is an :term:`asynchronous iterable`.
-
- See also :ref:`sync-async-spider-middleware` and
- :ref:`universal-spider-middleware`.
-
- .. versionadded:: 2.7
+ method of :ref:`spider middlewares `, which
+ *must* be defined as an :term:`asynchronous generator` except in
+ :ref:`universal spider middlewares `.
- The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method
of :ref:`spider middlewares `, which *must* be
@@ -63,6 +50,10 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
- :ref:`Signal handlers that support deferreds `.
+- Methods of :ref:`download handlers `.
+
+ .. versionadded:: 2.14
+
.. _coroutine-deferred-apis:
@@ -74,50 +65,29 @@ In addition to native coroutine APIs Scrapy has some APIs that return a
function that returns a :class:`~twisted.internet.defer.Deferred` object. These
APIs are also asynchronous but don't yet support native ``async def`` syntax.
In the future we plan to add support for the ``async def`` syntax to these APIs
-or replace them with other APIs where changing the existing ones is
+or replace them with other APIs where changing the existing ones isn't
possible.
-The following Scrapy methods return :class:`~twisted.internet.defer.Deferred`
-objects (this list is not complete as it only includes methods that we think
-may be useful for user code):
+These APIs have a coroutine-based implementation and a Deferred-based one:
- :class:`scrapy.crawler.Crawler`:
- - :meth:`~scrapy.crawler.Crawler.crawl`
+ - :meth:`~scrapy.crawler.Crawler.crawl_async` (coroutine-based) and
+ :meth:`~scrapy.crawler.Crawler.crawl` (Deferred-based): the former
+ may be inconvenient to use in Deferred-based code so both are available,
+ this may change in a future Scrapy version.
- - :meth:`~scrapy.crawler.Crawler.stop`
-
-- :class:`scrapy.crawler.CrawlerRunner` (also inherited by
- :class:`scrapy.crawler.CrawlerProcess`):
-
- - :meth:`~scrapy.crawler.CrawlerRunner.crawl`
-
- - :meth:`~scrapy.crawler.CrawlerRunner.stop`
-
- - :meth:`~scrapy.crawler.CrawlerRunner.join`
-
-- :class:`scrapy.core.engine.ExecutionEngine`:
-
- - :meth:`~scrapy.core.engine.ExecutionEngine.download`
-
-- :class:`scrapy.signalmanager.SignalManager`:
-
- - :meth:`~scrapy.signalmanager.SignalManager.send_catch_log_deferred`
-
-- :class:`~scrapy.mail.MailSender`
-
- - :meth:`~scrapy.mail.MailSender.send`
+- :class:`scrapy.crawler.AsyncCrawlerRunner` and its subclass
+ :class:`scrapy.crawler.AsyncCrawlerProcess` (coroutine-based) and
+ :class:`scrapy.crawler.CrawlerRunner` and its subclass
+ :class:`scrapy.crawler.CrawlerProcess` (Deferred-based): the former
+ doesn't support non-default reactors and so the latter should be used
+ with those.
The following user-supplied methods can return
:class:`~twisted.internet.defer.Deferred` objects (the methods that can also
return coroutines are listed in :ref:`coroutine-support`):
-- Custom download handlers (see :setting:`DOWNLOAD_HANDLERS`):
-
- - ``download_request()``
-
- - ``close()``
-
- Custom downloader implementations (see :setting:`DOWNLOADER`):
- ``fetch()``
@@ -156,19 +126,11 @@ wrapping a :class:`~twisted.internet.defer.Deferred` object into a
:class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for
more information about this.
-For example:
-
-- The :meth:`ExecutionEngine.download()
- ` method returns a
- :class:`~twisted.internet.defer.Deferred` object that fires with the
- downloaded response. You can use this object directly in Deferred-based
- code or convert it into a :class:`~asyncio.Future` object with
- :func:`~scrapy.utils.defer.maybe_deferred_to_future`.
-- A custom download handler needs to define a ``download_request()`` method
- that returns a :class:`~twisted.internet.defer.Deferred` object. You can
- write a method that works with Deferreds and returns one directly, or you
- can write a coroutine and convert it into a function that returns a
- Deferred with :func:`~scrapy.utils.defer.deferred_f_from_coro_f`.
+For example: a custom scheduler needs to define an ``open()`` method that can
+return a :class:`~twisted.internet.defer.Deferred` object. You can write a
+method that works with Deferreds and returns one directly, or you can write a
+coroutine and convert it into a function that returns a Deferred with
+:func:`~scrapy.utils.defer.deferred_f_from_coro_f`.
General usage
@@ -191,7 +153,7 @@ shorter and cleaner:
adapter["field"] = data
return item
- def process_item(self, item, spider):
+ def process_item(self, item):
adapter = ItemAdapter(item)
dfd = db.get_some_data(adapter["id"])
dfd.addCallback(self._update_item, item)
@@ -205,7 +167,7 @@ becomes:
class DbPipeline:
- async def process_item(self, item, spider):
+ async def process_item(self, item):
adapter = ItemAdapter(item)
adapter["field"] = await db.get_some_data(adapter["id"])
return item
@@ -244,13 +206,15 @@ This means you can use many useful Python libraries providing such code:
Common use cases for asynchronous code include:
* requesting data from websites, databases and other services (in
- :meth:`~scrapy.spiders.Spider.start`, callbacks, pipelines and
+ :meth:`~scrapy.Spider.start`, callbacks, pipelines and
middlewares);
* storing data in databases (in pipelines and middlewares);
* delaying the spider initialization until some external event (in the
:signal:`spider_opened` handler);
-* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download`
- (see :ref:`the screenshot pipeline example`).
+* calling asynchronous Scrapy methods like
+ :meth:`ExecutionEngine.download_async()
+ ` (see :ref:`the
+ screenshot pipeline example `).
.. _aio-libs: https://github.com/aio-libs
@@ -266,7 +230,6 @@ within a spider callback:
.. code-block:: python
from scrapy import Spider, Request
- from scrapy.utils.defer import maybe_deferred_to_future
class SingleRequestSpider(Spider):
@@ -275,8 +238,9 @@ within a spider callback:
async def parse(self, response, **kwargs):
additional_request = Request("https://example.org/price")
- deferred = self.crawler.engine.download(additional_request)
- additional_response = await maybe_deferred_to_future(deferred)
+ additional_response = await self.crawler.engine.download_async(
+ additional_request
+ )
yield {
"h1": response.css("h1").get(),
"price": additional_response.css("#price").get(),
@@ -286,9 +250,9 @@ You can also send multiple requests in parallel:
.. code-block:: python
+ import asyncio
+
from scrapy import Spider, Request
- from scrapy.utils.defer import maybe_deferred_to_future
- from twisted.internet.defer import DeferredList
class MultipleRequestsSpider(Spider):
@@ -300,150 +264,13 @@ You can also send multiple requests in parallel:
Request("https://example.com/price"),
Request("https://example.com/color"),
]
- deferreds = []
+ tasks = []
for r in additional_requests:
- deferred = self.crawler.engine.download(r)
- deferreds.append(deferred)
- responses = await maybe_deferred_to_future(DeferredList(deferreds))
+ task = self.crawler.engine.download_async(r)
+ tasks.append(task)
+ responses = await asyncio.gather(*tasks)
yield {
"h1": response.css("h1::text").get(),
- "price": responses[0][1].css(".price::text").get(),
- "price2": responses[1][1].css(".color::text").get(),
+ "price": responses[0].css(".price::text").get(),
+ "color": responses[1].css(".color::text").get(),
}
-
-
-.. _sync-async-spider-middleware:
-
-Mixing synchronous and asynchronous spider middlewares
-======================================================
-
-.. versionadded:: 2.7
-
-The output of a :class:`~scrapy.Request` callback is passed as the ``result``
-parameter to the
-:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
-of the first :ref:`spider middleware ` from the
-:ref:`list of active spider middlewares `.
-Then the output of that ``process_spider_output`` method is passed to the
-``process_spider_output`` method of the next spider middleware, and so on for
-every active spider middleware.
-
-Scrapy supports mixing :ref:`coroutine methods ` and synchronous methods
-in this chain of calls.
-
-However, if any of the ``process_spider_output`` methods is defined as a
-synchronous method, and the previous ``Request`` callback or
-``process_spider_output`` method is a coroutine, there are some drawbacks to
-the asynchronous-to-synchronous conversion that Scrapy does so that the
-synchronous ``process_spider_output`` method gets a synchronous iterable as its
-``result`` parameter:
-
-- The whole output of the previous ``Request`` callback or
- ``process_spider_output`` method is awaited at this point.
-
-- If an exception raises while awaiting the output of the previous
- ``Request`` callback or ``process_spider_output`` method, none of that
- output will be processed.
-
- This contrasts with the regular behavior, where all items yielded before
- an exception raises are processed.
-
-Asynchronous-to-synchronous conversions are supported for backward
-compatibility, but they are deprecated and will stop working in a future
-version of Scrapy.
-
-To avoid asynchronous-to-synchronous conversions, when defining ``Request``
-callbacks as coroutine methods or when using spider middlewares whose
-``process_spider_output`` method is an :term:`asynchronous generator`, all
-active spider middlewares must either have their ``process_spider_output``
-method defined as an asynchronous generator or :ref:`define a
-process_spider_output_async method `.
-
-.. _sync-async-spider-middleware-users:
-
-For middleware users
---------------------
-
-If you have asynchronous callbacks or use asynchronous-only spider middlewares
-you should make sure the asynchronous-to-synchronous conversions
-:ref:`described above ` don't happen. To do this,
-make sure all spider middlewares you use support asynchronous spider output.
-Even if you don't have asynchronous callbacks and don't use asynchronous-only
-spider middlewares in your project, it's still a good idea to make sure all
-middlewares you use support asynchronous spider output, so that it will be easy
-to start using asynchronous callbacks in the future. Because of this, Scrapy
-logs a warning when it detects a synchronous-only spider middleware.
-
-If you want to update middlewares you wrote, see the :ref:`following section
-`. If you have 3rd-party middlewares that
-aren't yet updated by their authors, you can :ref:`subclass `
-them to make them :ref:`universal ` and use the
-subclasses in your projects.
-
-.. _sync-async-spider-middleware-authors:
-
-For middleware authors
-----------------------
-
-If you have a spider middleware that defines a synchronous
-``process_spider_output`` method, you should update it to support asynchronous
-spider output for :ref:`better compatibility `,
-even if you don't yet use it with asynchronous callbacks, especially if you
-publish this middleware for other people to use. You have two options for this:
-
-1. Make the middleware asynchronous, by making the ``process_spider_output``
- method an :term:`asynchronous generator`.
-2. Make the middleware universal, as described in the :ref:`next section
- `.
-
-If your middleware won't be used in projects with synchronous-only middlewares,
-e.g. because it's an internal middleware and you know that all other
-middlewares in your projects are already updated, it's safe to choose the first
-option. Otherwise, it's better to choose the second option.
-
-.. _universal-spider-middleware:
-
-Universal spider middlewares
-----------------------------
-
-.. versionadded:: 2.7
-
-To allow writing a spider middleware that supports asynchronous execution of
-its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding
-:ref:`asynchronous-to-synchronous conversions `)
-while maintaining support for older Scrapy versions, you may define
-``process_spider_output`` as a synchronous method and define an
-:term:`asynchronous generator` version of that method with an alternative name:
-``process_spider_output_async``.
-
-For example:
-
-.. code-block:: python
-
- class UniversalSpiderMiddleware:
- def process_spider_output(self, response, result, spider):
- for r in result:
- # ... do something with r
- yield r
-
- async def process_spider_output_async(self, response, result, spider):
- async for r in result:
- # ... do something with r
- yield r
-
-.. note:: This is an interim measure to allow, for a time, to write code that
- works in Scrapy 2.7 and later without requiring
- asynchronous-to-synchronous conversions, and works in earlier Scrapy
- versions as well.
-
- In some future version of Scrapy, however, this feature will be
- deprecated and, eventually, in a later version of Scrapy, this
- feature will be removed, and all spider middlewares will be expected
- to define their ``process_spider_output`` method as an asynchronous
- generator.
-
-Since 2.13.0, Scrapy provides a base class,
-:class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which implements
-the ``process_spider_output()`` and ``process_spider_output_async()`` methods,
-so instead of duplicating the processing code you can override the
-``get_processed_request()`` and/or the ``get_processed_item()`` method.
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index 89a4d32d8..a5ffe00f1 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -246,7 +246,6 @@ also request each page to get every quote on the site:
.. code-block:: python
import scrapy
- import json
class QuoteSpider(scrapy.Spider):
@@ -256,7 +255,7 @@ also request each page to get every quote on the site:
start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"]
def parse(self, response):
- data = json.loads(response.text)
+ data = response.json()
for quote in data["quotes"]:
yield {"quote": quote["text"]}
if data["has_next"]:
@@ -280,7 +279,7 @@ In more complex websites, it could be difficult to easily reproduce the
requests, as we could need to add ``headers`` or ``cookies`` to make it work.
In those cases you can export the requests in `cURL `_
format, by right-clicking on each of them in the network tool and using the
-:meth:`~scrapy.Request.from_curl()` method to generate an equivalent
+:meth:`~scrapy.Request.from_curl` method to generate an equivalent
request:
.. code-block:: python
@@ -317,4 +316,3 @@ to identifying the correct request and replicating it in your spider.
.. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll
.. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10
.. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
-
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
new file mode 100644
index 000000000..e0501c169
--- /dev/null
+++ b/docs/topics/download-handlers.rst
@@ -0,0 +1,385 @@
+.. _topics-download-handlers:
+
+=================
+Download handlers
+=================
+
+Download handlers are Scrapy :ref:`components ` used to
+download :ref:`requests ` and produce responses from
+them.
+
+Using download handlers
+=======================
+
+The :setting:`DOWNLOAD_HANDLERS_BASE` and :setting:`DOWNLOAD_HANDLERS` settings
+tell Scrapy which handler is responsible for a given URL scheme. Their values
+are merged into a mapping from scheme names to handler classes. When Scrapy
+initializes it creates instances of all configured download handlers (except
+for :ref:`lazy ones `) and stores them in a similar
+mapping. When Scrapy needs to download a request it extracts the scheme from
+its URL, finds the handler for this scheme, passes the request to it and gets a
+response from it. If there is no handler for the scheme, the request is not
+downloaded and a :exc:`~scrapy.exceptions.NotSupported` exception is raised.
+
+The :setting:`DOWNLOAD_HANDLERS_BASE` setting contains the default mapping of
+handlers. You can use the :setting:`DOWNLOAD_HANDLERS` setting to add handlers
+for additional schemes and to replace or disable default ones:
+
+.. code-block:: python
+
+ DOWNLOAD_HANDLERS = {
+ # disable support for ftp:// requests
+ "ftp": None,
+ # replace the default one for http://
+ "http": "my.download_handlers.HttpHandler",
+ # http:// and https:// are different schemes,
+ # even though they may use the same handler
+ "https": "my.download_handlers.HttpHandler",
+ # support for any custom scheme can be added
+ "sftp": "my.download_handlers.SftpHandler",
+ }
+
+.. seealso:: :ref:`security-unencrypted-protocols` and
+ :ref:`security-local-resources`, for the security implications of the
+ default ``http``, ``ftp``, ``file`` and ``data`` handlers.
+
+Replacing HTTP(S) download handlers
+-----------------------------------
+
+While Scrapy provides a default handler for ``http`` and ``https`` schemes,
+users may want to use a different handler, provided by Scrapy or by some
+3rd-party package. There are several considerations to keep in mind related to
+this.
+
+First of all, as ``http`` and ``https`` are separate schemes, they need
+separate entries in the :setting:`DOWNLOAD_HANDLERS` setting, even though it's
+likely that the same handler class will be used for both schemes.
+
+Additionally, some of the Scrapy settings, like :setting:`DOWNLOAD_MAXSIZE`,
+are honored by the default HTTP(S) handler but not necessarily by alternative
+ones. The same may apply to other Scrapy features, e.g. the
+:signal:`bytes_received` and :signal:`headers_received` signals.
+
+.. _lazy-download-handlers:
+
+Lazy instantiation of download handlers
+---------------------------------------
+
+A download handler can be marked as "lazy" by setting its ``lazy`` class
+attribute to ``True``. Such handlers are only instantiated when they need to
+download their first request. This may be useful when the instantiation is slow
+or requires dependencies that are not always available, and the handler is not
+needed on every spider run. For example, :class:`the built-in S3 handler
+<.S3DownloadHandler>` is lazy.
+
+Writing your own download handler
+=================================
+
+A download handler is a :ref:`component ` that defines
+the following API:
+
+.. autoclass:: scrapy.core.downloader.handlers.DownloadHandlerProtocol
+ :members:
+
+An optional base class for custom handlers is provided:
+
+.. autoclass:: scrapy.core.downloader.handlers.base.BaseDownloadHandler
+ :members:
+ :undoc-members:
+ :exclude-members: close, download_request, lazy
+
+.. _download-handlers-exceptions:
+
+Exceptions raised by download handlers
+======================================
+
+.. versionadded:: 2.15.0
+
+The built-in download handlers raise Scrapy-specific exceptions instead of
+implementation-specific ones, so that code that handles these exceptions can be
+written in a generic way. We recommend custom download handlers to also use
+these exceptions.
+
+.. autoexception:: scrapy.exceptions.CannotResolveHostError
+
+.. autoexception:: scrapy.exceptions.DownloadCancelledError
+
+.. autoexception:: scrapy.exceptions.DownloadConnectionRefusedError
+
+.. autoexception:: scrapy.exceptions.DownloadFailedError
+
+.. autoexception:: scrapy.exceptions.DownloadTimeoutError
+
+.. autoexception:: scrapy.exceptions.ResponseDataLossError
+
+.. autoexception:: scrapy.exceptions.UnsupportedURLSchemeError
+
+.. _download-handlers-ref:
+
+Built-in HTTP download handlers reference
+=========================================
+
+Scrapy ships several handlers for HTTP and HTTPS requests. While all of them
+support basic features, they may differ in support of specific Scrapy features
+and settings and HTTP protocol features. See the documentation of specific
+handlers and specific settings for more information. Additionally, as the
+underlying HTTP client implementations differ between handlers, the behavior of
+specific websites may be different when doing the same Scrapy requests but
+using different handlers.
+
+Here is a comparison of some features of the built-in HTTP handlers, see the
+individual handler docs for more differences:
+
+================== ================= ===================== ====================
+Feature H2DownloadHandler HTTP11DownloadHandler HttpxDownloadHandler
+================== ================= ===================== ====================
+Requires asyncio No No Yes
+Requires a reactor Yes Yes No
+HTTP/1.1 No Yes Yes
+HTTP/2 Yes No Yes
+TLS implementation ``cryptography`` ``cryptography`` Stdlib ``ssl``
+HTTP proxies No Yes Yes
+SOCKS proxies No No Yes
+================== ================= ===================== ====================
+
+You can find additional HTTP download handlers in the
+scrapy-download-handlers-incubator_ package. This package is made by the Scrapy
+developers and contains experimental handlers that may be included in some
+later Scrapy version but can already be used. Please refer to the documentation
+of this package for more information.
+
+.. _scrapy-download-handlers-incubator: https://github.com/scrapy-plugins/scrapy-download-handlers-incubator
+
+.. _twisted-http2-handler:
+
+H2DownloadHandler
+-----------------
+
+.. note:: Requires the :ref:`twisted-http2 ` extra.
+
+.. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler
+
+| Supported scheme: ``https``.
+| :ref:`Lazy `: yes.
+| :ref:`Requires asyncio support `: no.
+| :ref:`Requires a Twisted reactor `: yes.
+
+This handler supports ``https://host/path`` URLs and uses the HTTP/2 protocol
+for them.
+
+It's implemented using :mod:`twisted.web.client` and the ``h2`` library.
+
+If you want to use this handler you need to replace the default one for the
+``https`` scheme:
+
+.. code-block:: python
+
+ DOWNLOAD_HANDLERS = {
+ "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",
+ }
+
+Features and limitations
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. warning::
+
+ This handler is experimental, and not yet recommended for production
+ environments. Future Scrapy versions may introduce related changes without
+ a deprecation period or warning.
+
+=========================== ================================================
+HTTP proxies No (not implemented)
+SOCKS proxies No (not supported by the library)
+HTTP/2 Yes
+``response.certificate`` :class:`twisted.internet.ssl.Certificate` object
+Per-request ``bindaddress`` Yes
+TLS implementation ``pyOpenSSL``/``cryptography``
+=========================== ================================================
+
+Other limitations:
+
+- No support for HTTP/1.1.
+
+- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`
+ to ``scrapy.resolver.CachingHostnameResolver``.
+
+- No support for the :signal:`bytes_received` and :signal:`headers_received`
+ signals.
+
+Known limitations of the HTTP/2 support:
+
+- No support for HTTP/2 Cleartext (h2c), since no major browser supports
+ HTTP/2 unencrypted (refer `http2 faq`_).
+
+- No setting to specify a maximum `frame size`_ larger than the default
+ value, 16384. Connections to servers that send a larger frame will fail.
+
+- No support for `server pushes`_, which are ignored.
+
+.. _frame size: https://datatracker.ietf.org/doc/html/rfc7540#section-4.2
+.. _http2 faq: https://http2.github.io/faq/#does-http2-require-encryption
+.. _server pushes: https://datatracker.ietf.org/doc/html/rfc7540#section-8.2
+
+HTTP11DownloadHandler
+---------------------
+
+.. autoclass:: scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler
+
+| Supported schemes: ``http``, ``https``.
+| :ref:`Lazy `: no.
+| :ref:`Requires asyncio support `: no.
+| :ref:`Requires a Twisted reactor `: yes.
+
+This handler supports ``http://host/path`` and ``https://host/path`` URLs and
+uses the HTTP/1.1 protocol for them.
+
+It's implemented using :mod:`twisted.web.client`.
+
+Features and limitations
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+=========================== ================================================
+HTTP proxies Yes
+SOCKS proxies No (not supported by the library)
+HTTP/2 No (implemented as a separate handler)
+``response.certificate`` :class:`twisted.internet.ssl.Certificate` object
+Per-request ``bindaddress`` Yes
+TLS implementation ``pyOpenSSL``/``cryptography``
+=========================== ================================================
+
+Other limitations:
+
+- IPv6 support requires setting :setting:`TWISTED_DNS_RESOLVER`
+ to ``scrapy.resolver.CachingHostnameResolver``.
+
+- HTTPS proxies to HTTPS destinations are not supported.
+
+.. _httpx-handler:
+
+HttpxDownloadHandler
+--------------------
+
+.. note:: Requires the :ref:`httpx ` extra.
+
+.. versionadded:: 2.15.0
+
+.. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler
+
+| Supported schemes: ``http``, ``https``.
+| :ref:`Lazy `: no.
+| :ref:`Requires asyncio support `: yes.
+| :ref:`Requires a Twisted reactor `: no.
+
+This handler supports ``http://host/path`` and ``https://host/path`` URLs and
+uses the HTTP/1.1 or HTTP/2 protocol for them.
+
+It's implemented using the httpx2_ library.
+
+.. _httpx2: https://httpx2.pydantic.dev/
+
+If you want to use this handler you need to replace the default ones for the
+``http`` and ``https`` schemes:
+
+.. code-block:: python
+
+ DOWNLOAD_HANDLERS = {
+ "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
+ "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
+ }
+
+Features and limitations
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. warning::
+
+ This handler is experimental, and not yet recommended for production
+ environments. Future Scrapy versions may introduce related changes without
+ a deprecation period or warning or even remove it altogether.
+
+=========================== =======================================
+HTTP proxies Yes
+SOCKS proxies Yes (SOCKS5)
+HTTP/2 Yes
+``response.certificate`` DER bytes
+Per-request ``bindaddress`` No (not supported by the library)
+TLS implementation Standard library ``ssl``
+=========================== =======================================
+
+Other limitations:
+
+- The handler creates a separate connection pool for each proxy URL (due to
+ limitations of ``httpx``) which may lead to higher resource usage when
+ using proxy rotation.
+
+.. setting:: HTTPX_HTTP2_ENABLED
+
+HTTPX_HTTP2_ENABLED
+^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: 2.17.0
+
+Default: ``False``
+
+Whether to enable HTTP/2 support in this handler.
+
+Built-in non-HTTP download handlers reference
+=============================================
+
+DataURIDownloadHandler
+----------------------
+
+.. autoclass:: scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler
+
+| Supported scheme: ``data``.
+| :ref:`Lazy `: no.
+| :ref:`Requires asyncio support `: no.
+| :ref:`Requires a Twisted reactor `: no.
+
+This handler supports RFC 2397 ``data:content/type;base64,`` data URIs.
+
+FileDownloadHandler
+-------------------
+
+.. autoclass:: scrapy.core.downloader.handlers.file.FileDownloadHandler
+
+| Supported scheme: ``file``.
+| :ref:`Lazy `: no.
+| :ref:`Requires asyncio support `: no.
+| :ref:`Requires a Twisted reactor `: no.
+
+This handler supports ``file:///path`` local file URIs. It doesn't
+support remote files.
+
+FTPDownloadHandler
+------------------
+
+.. autoclass:: scrapy.core.downloader.handlers.ftp.FTPDownloadHandler
+
+| Supported scheme: ``ftp``.
+| :ref:`Lazy `: no.
+| :ref:`Requires asyncio support `: no.
+| :ref:`Requires a Twisted reactor `: yes.
+
+This handler supports ``ftp://host/path`` FTP URIs.
+
+It's implemented using :mod:`twisted.protocols.ftp`.
+
+.. _s3-handler:
+
+S3DownloadHandler
+-----------------
+
+.. note:: Requires the :ref:`s3 ` extra.
+
+.. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler
+
+| Supported scheme: ``s3``.
+| :ref:`Lazy `: yes.
+| :ref:`Requires asyncio support `: no.
+| :ref:`Requires a Twisted reactor `: no.
+
+This handler supports ``s3://bucket/path`` S3 URIs.
+
+It's implemented using the botocore_ library.
+
+.. _botocore: https://github.com/boto/botocore
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 60b6aab78..fcfe7fd29 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -68,9 +68,10 @@ defines one or more of these methods:
.. class:: DownloaderMiddleware
- .. note:: Any of the downloader middleware methods may also return a deferred.
+ .. note:: Any of the downloader middleware methods may be defined as a
+ coroutine function (``async def``).
- .. method:: process_request(request, spider)
+ .. method:: process_request(request)
This method is called for each request that goes through the download
middleware.
@@ -102,10 +103,7 @@ defines one or more of these methods:
:param request: the request being processed
:type request: :class:`~scrapy.Request` object
- :param spider: the spider for which this request is intended
- :type spider: :class:`~scrapy.Spider` object
-
- .. method:: process_response(request, response, spider)
+ .. method:: process_response(request, response)
:meth:`process_response` should either: return a :class:`~scrapy.http.Response`
object, return a :class:`~scrapy.Request` object or
@@ -129,14 +127,12 @@ defines one or more of these methods:
:param response: the response being processed
:type response: :class:`~scrapy.http.Response` object
- :param spider: the spider for which this response is intended
- :type spider: :class:`~scrapy.Spider` object
+ .. method:: process_exception(request, exception)
- .. method:: process_exception(request, exception, spider)
-
- Scrapy calls :meth:`process_exception` when a download handler
- or a :meth:`process_request` (from a downloader middleware) raises an
- exception (including an :exc:`~scrapy.exceptions.IgnoreRequest` exception)
+ Scrapy calls :meth:`process_exception` when a :ref:`download handler
+ ` or a :meth:`process_request` (from a
+ downloader middleware) raises an exception (including an
+ :exc:`~scrapy.exceptions.IgnoreRequest` exception).
:meth:`process_exception` should return: either ``None``,
a :class:`~scrapy.http.Response` object, or a :class:`~scrapy.Request` object.
@@ -160,9 +156,6 @@ defines one or more of these methods:
:param exception: the raised exception
:type exception: an ``Exception`` object
- :param spider: the spider for which this request is intended
- :type spider: :class:`~scrapy.Spider` object
-
.. _topics-downloader-middleware-ref:
Built-in downloader middleware reference
@@ -298,13 +291,12 @@ DownloadTimeoutMiddleware
.. class:: DownloadTimeoutMiddleware
This middleware sets the download timeout for requests specified in the
- :setting:`DOWNLOAD_TIMEOUT` setting or :attr:`download_timeout`
- spider attribute.
+ :setting:`DOWNLOAD_TIMEOUT` setting.
.. note::
- You can also set download timeout per-request using
- :reqmeta:`download_timeout` Request.meta key; this is supported
+ You can also set download timeout per-request using the
+ :reqmeta:`download_timeout` :attr:`.Request.meta` key; this is supported
even when DownloadTimeoutMiddleware is disabled.
HttpAuthMiddleware
@@ -315,26 +307,15 @@ HttpAuthMiddleware
.. class:: HttpAuthMiddleware
- This middleware authenticates all requests generated from certain spiders
- using `Basic access authentication`_ (aka. HTTP auth).
+ This middleware authenticates requests using `Basic access authentication`_
+ (aka. HTTP auth).
- To enable HTTP authentication for a spider, set the ``http_user`` and
- ``http_pass`` spider attributes to the authentication data and the
- ``http_auth_domain`` spider attribute to the domain which requires this
- authentication (its subdomains will be also handled in the same way).
- You can set ``http_auth_domain`` to ``None`` to enable the
- authentication for all requests but you risk leaking your authentication
- credentials to unrelated domains.
+ Use the :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS`, and
+ :setting:`HTTPAUTH_DOMAIN` settings to configure it. You can also override
+ the credentials per request via :attr:`~scrapy.Request.meta` keys
+ :reqmeta:`http_user`, :reqmeta:`http_pass`, and :reqmeta:`http_auth_domain`.
- .. warning::
- In previous Scrapy versions HttpAuthMiddleware sent the authentication
- data with all requests, which is a security problem if the spider
- makes requests to several different domains. Currently if the
- ``http_auth_domain`` attribute is not set, the middleware will use the
- domain of the first request, which will work for some spiders but not
- for others. In the future the middleware will produce an error instead.
-
- Example:
+ Example using settings (e.g. in :attr:`~scrapy.Spider.custom_settings`):
.. code-block:: python
@@ -342,13 +323,70 @@ HttpAuthMiddleware
class SomeIntranetSiteSpider(CrawlSpider):
- http_user = "someuser"
- http_pass = "somepass"
- http_auth_domain = "intranet.example.com"
name = "intranet.example.com"
+ custom_settings = {
+ "HTTPAUTH_USER": "someuser",
+ "HTTPAUTH_PASS": "somepass",
+ "HTTPAUTH_DOMAIN": "intranet.example.com",
+ }
# .. rest of the spider code omitted ...
+ Example using per-request meta:
+
+ .. code-block:: python
+
+ async def start(self):
+ yield Request(
+ "https://intranet.example.com/protected/",
+ meta={
+ "http_user": "someuser",
+ "http_pass": "somepass",
+ "http_auth_domain": "intranet.example.com",
+ },
+ )
+
+.. setting:: HTTPAUTH_USER
+
+HTTPAUTH_USER
+~~~~~~~~~~~~~
+
+.. versionadded:: 2.17.0
+
+Default: ``""``
+
+The username to use for HTTP basic authentication, applied to all requests
+whose URL matches :setting:`HTTPAUTH_DOMAIN`.
+
+.. setting:: HTTPAUTH_PASS
+
+HTTPAUTH_PASS
+~~~~~~~~~~~~~
+
+.. versionadded:: 2.17.0
+
+Default: ``""``
+
+The password to use for HTTP basic authentication.
+
+.. setting:: HTTPAUTH_DOMAIN
+
+HTTPAUTH_DOMAIN
+~~~~~~~~~~~~~~~
+
+.. versionadded:: 2.17.0
+
+Default: ``None``
+
+The domain (and its subdomains) to which HTTP basic authentication credentials
+are sent. Set to ``None`` to send credentials with all requests, but be aware
+that this risks leaking credentials to unrelated domains.
+
+This setting must be explicitly configured whenever :setting:`HTTPAUTH_USER`
+or :setting:`HTTPAUTH_PASS` is set.
+
+.. seealso:: :ref:`security-credential-leakage`
+
.. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication
@@ -467,7 +505,7 @@ Filesystem storage backend (default)
* ``response_body`` - the plain response body
- * ``response_headers`` - the request headers (in raw HTTP format)
+ * ``response_headers`` - the response headers (in raw HTTP format)
* ``meta`` - some metadata of this cache resource in Python ``repr()``
format (grep-friendly format)
@@ -509,7 +547,7 @@ defines the methods described below.
.. method:: open_spider(spider)
This method gets called after a spider has been opened for crawling. It handles
- the :signal:`open_spider ` signal.
+ the :signal:`spider_opened` signal.
:param spider: the spider which has been opened
:type spider: :class:`~scrapy.Spider` object
@@ -517,7 +555,7 @@ defines the methods described below.
.. method:: close_spider(spider)
This method gets called after a spider has been closed. It handles
- the :signal:`close_spider ` signal.
+ the :signal:`spider_closed` signal.
:param spider: the spider which has been closed
:type spider: :class:`~scrapy.Spider` object
@@ -526,6 +564,10 @@ defines the methods described below.
Return response if present in cache, or ``None`` otherwise.
+ If this method raises an exception, e.g. because the cache entry is
+ corrupted, the middleware logs a warning and handles the request as a
+ cache miss.
+
:param spider: the spider which generated the request
:type spider: :class:`~scrapy.Spider` object
@@ -553,8 +595,8 @@ In order to use your storage backend, set:
HTTPCache middleware settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-The :class:`HttpCacheMiddleware` can be configured through the following
-settings:
+:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` can be
+configured through the following settings:
.. setting:: HTTPCACHE_ENABLED
@@ -689,6 +731,8 @@ We assume that the spider will not issue Cache-Control directives
in requests unless it actually needs them, so directives in requests are
not filtered.
+.. _http-compression:
+
HttpCompressionMiddleware
-------------------------
@@ -700,14 +744,12 @@ HttpCompressionMiddleware
This middleware allows compressed (gzip, deflate) traffic to be
sent/received from web sites.
- This middleware also supports decoding `brotli-compressed`_ as well as
- `zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is
- installed, respectively.
+ This middleware also supports decoding `brotli-compressed`_ responses with
+ the :ref:`brotli ` extra, and `zstd-compressed`_
+ responses with the :ref:`zstd ` extra.
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
-.. _brotli: https://pypi.org/project/Brotli/
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
-.. _zstandard: https://pypi.org/project/zstandard/
HttpCompressionMiddleware Settings
@@ -734,7 +776,7 @@ HttpProxyMiddleware
.. class:: HttpProxyMiddleware
This middleware sets the HTTP proxy to use for requests, by setting the
- ``proxy`` meta value for :class:`~scrapy.Request` objects.
+ :reqmeta:`proxy` meta value for :class:`~scrapy.Request` objects.
Like the Python standard library module :mod:`urllib.request`, it obeys
the following environment variables:
@@ -743,16 +785,39 @@ HttpProxyMiddleware
* ``https_proxy``
* ``no_proxy``
- You can also set the meta key ``proxy`` per-request, to a value like
+ You can also set the meta key :reqmeta:`proxy` per-request, to a value like
``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``.
Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
environment variables, and it will also ignore ``no_proxy`` environment variable.
+.. note::
+
+ Handling of this meta key needs to be implemented inside the :ref:`download
+ handler `, so it's not guaranteed to be supported
+ by all 3rd-party handlers. It's currently unsupported by
+ :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`.
+
+.. note::
+
+ Usually a proxy URL uses the ``http://`` scheme. More rarely, it uses the
+ ``https://`` one. While both kinds of proxy URLs can be used with both HTTP
+ and HTTPS destination URLs, the specifics of the network exchange are
+ different for all 4 cases and it's possible that HTTPS proxies are fully or
+ partially unsupported by a given download handler. Currently,
+ :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
+ supports HTTPS proxies only for HTTP destinations.
+
+.. note::
+
+ If the download handler supports it, you can use a SOCKS proxy URL (e.g.
+ ``socks5://username:password@some_proxy_server:port``).
+ :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`
+ supports SOCKS proxies while other built-in handlers don't.
+
HttpProxyMiddleware settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
.. setting:: HTTPPROXY_ENABLED
-.. setting:: HTTPPROXY_AUTH_ENCODING
HTTPPROXY_ENABLED
^^^^^^^^^^^^^^^^^
@@ -761,6 +826,8 @@ Default: ``True``
Whether or not to enable the :class:`HttpProxyMiddleware`.
+.. setting:: HTTPPROXY_AUTH_ENCODING
+
HTTPPROXY_AUTH_ENCODING
^^^^^^^^^^^^^^^^^^^^^^^
@@ -805,9 +872,9 @@ OffsiteMiddleware
.. reqmeta:: allow_offsite
If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to
- ``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then
- the OffsiteMiddleware will allow the request even if its domain is not listed
- in allowed domains.
+ ``True`` or :attr:`Request.meta ` has ``allow_offsite``
+ set to ``True``, then the OffsiteMiddleware will allow the request even if
+ its domain is not listed in allowed domains.
RedirectMiddleware
------------------
@@ -922,14 +989,10 @@ Whether the Meta Refresh middleware will be enabled.
METAREFRESH_IGNORE_TAGS
^^^^^^^^^^^^^^^^^^^^^^^
-Default: ``[]``
+Default: ``["noscript"]``
Meta tags within these tags are ignored.
-.. versionchanged:: 2.0
- The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
- ``["script", "noscript"]`` to ``[]``.
-
.. versionchanged:: 2.11.2
The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
``[]`` to ``["noscript"]``.
@@ -956,17 +1019,6 @@ RetryMiddleware
A middleware to retry failed requests that are potentially caused by
temporary problems such as a connection timeout or HTTP 500 error.
-Failed pages are collected on the scraping process and rescheduled at the
-end, once the spider has finished crawling all regular (non failed) pages.
-
-The :class:`RetryMiddleware` can be configured through the following
-settings (see the settings documentation for more info):
-
-* :setting:`RETRY_ENABLED`
-* :setting:`RETRY_TIMES`
-* :setting:`RETRY_HTTP_CODES`
-* :setting:`RETRY_EXCEPTIONS`
-
.. reqmeta:: dont_retry
If :attr:`Request.meta ` has ``dont_retry`` key
@@ -1025,16 +1077,15 @@ RETRY_EXCEPTIONS
Default::
[
- 'twisted.internet.defer.TimeoutError',
- 'twisted.internet.error.TimeoutError',
- 'twisted.internet.error.DNSLookupError',
- 'twisted.internet.error.ConnectionRefusedError',
+ 'scrapy.exceptions.CannotResolveHostError',
+ 'scrapy.exceptions.DownloadConnectionRefusedError',
+ 'scrapy.exceptions.DownloadFailedError',
+ 'scrapy.exceptions.DownloadTimeoutError',
+ 'scrapy.exceptions.ResponseDataLossError',
'twisted.internet.error.ConnectionDone',
'twisted.internet.error.ConnectError',
'twisted.internet.error.ConnectionLost',
- 'twisted.internet.error.TCPTimedOutError',
- 'twisted.web.client.ResponseFailed',
- IOError,
+ OSError,
'scrapy.core.downloader.handlers.http11.TunnelError',
]
@@ -1048,6 +1099,23 @@ has been exceeded (see :setting:`RETRY_TIMES`). To learn about uncaught
exception propagation, see
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_exception`.
+.. setting:: RETRY_GIVE_UP_LOG_LEVEL
+
+RETRY_GIVE_UP_LOG_LEVEL
+^^^^^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: 2.17.0
+
+Default: ``"ERROR"``
+
+:ref:`Logging level ` used for the message logged when a request
+exceeds its retries.
+
+Can be a level name (e.g. ``"WARNING"``) or a number (e.g. ``logging.WARNING``
+or ``30``).
+
+See also: :reqmeta:`give_up_log_level`, :func:`get_retry_request`.
+
.. setting:: RETRY_PRIORITY_ADJUST
RETRY_PRIORITY_ADJUST
@@ -1109,7 +1177,7 @@ Parsers vary in several aspects:
* Support for wildcard matching
-* Usage of `length based rule `_:
+* Usage of `length based rule `_:
in particular for ``Allow`` and ``Disallow`` directives, where the most
specific rule based on the length of the path trumps the less specific
(shorter) rule
@@ -1127,7 +1195,7 @@ Based on `Protego `_:
* implemented in Python
* is compliant with `Google's Robots.txt Specification
- `_
+ `_
* supports wildcard matching
@@ -1147,9 +1215,9 @@ Based on :class:`~urllib.robotparser.RobotFileParser`:
* is compliant with `Martijn Koster's 1996 draft specification
`_
-* lacks support for wildcard matching
+* lacks support for wildcard matching (before Python 3.14.5)
-* doesn't use the length based rule
+* doesn't use the length based rule (before Python 3.14.5)
It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0.
@@ -1175,8 +1243,7 @@ Based on `Robotexclusionrulesparser ` extra.
* Set :setting:`ROBOTSTXT_PARSER` setting to
``scrapy.robotstxt.RerpRobotParser``
@@ -1220,9 +1287,8 @@ UserAgentMiddleware
.. class:: UserAgentMiddleware
- Middleware that allows spiders to override the default user agent.
+ Middleware that sets the ``User-Agent`` header.
- In order for a spider to override the default user agent, its ``user_agent``
- attribute must be set.
+ The header value is taken from the :setting:`USER_AGENT` setting.
.. _DBM: https://en.wikipedia.org/wiki/Dbm
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 65270433f..30b6536c7 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -83,10 +83,10 @@ request with Scrapy.
It might be enough to yield a :class:`~scrapy.Request` with the same HTTP
method and URL. However, you may also need to reproduce the body, headers and
-form parameters (see :class:`~scrapy.FormRequest`) of that request.
+form parameters (see :ref:`form`) of that request.
As all major browsers allow to export the requests in curl_ format, Scrapy
-incorporates the method :meth:`~scrapy.Request.from_curl()` to generate an equivalent
+incorporates the method :meth:`~scrapy.Request.from_curl` to generate an equivalent
:class:`~scrapy.Request` from a cURL command. To get more information
visit :ref:`request from curl ` inside the network
tool section.
@@ -111,6 +111,8 @@ you may use `curl2scrapy `_.
Handling different response formats
===================================
+.. skip: start
+
Once you have a response with the desired data, how you extract the desired
data from it depends on the type of response:
@@ -131,7 +133,7 @@ data from it depends on the type of response:
.. code-block:: python
- selector = Selector(data["html"])
+ selector = Selector(text=data["html"])
- If the response is JavaScript, or HTML with a ```` element
containing the desired data, see :ref:`topics-parsing-javascript`.
@@ -157,11 +159,15 @@ data from it depends on the type of response:
Otherwise, you might need to convert the SVG code into a raster image, and
:ref:`handle that raster image `.
+.. skip: end
+
.. _topics-parsing-javascript:
Parsing JavaScript code
=======================
+.. skip: start
+
If the desired data is hardcoded in JavaScript, you first need to get the
JavaScript code:
@@ -220,6 +226,8 @@ data from it:
>>> selector.css('var[name="data"]').get()
''
+.. skip: end
+
.. _topics-headless-browsing:
Using a headless browser
@@ -242,6 +250,7 @@ it is possible to integrate ``asyncio``-based libraries which handle headless br
One such library is `playwright-python`_ (an official Python port of `playwright`_).
The following is a simple snippet to illustrate its usage within a Scrapy spider:
+.. skip: next
.. code-block:: python
import scrapy
@@ -265,16 +274,13 @@ However, using `playwright-python`_ directly as in the above example
circumvents most of the Scrapy components (middlewares, dupefilter, etc).
We recommend using `scrapy-playwright`_ for a better integration.
-.. _AJAX: https://en.wikipedia.org/wiki/Ajax_%28programming%29
.. _CSS: https://en.wikipedia.org/wiki/Cascading_Style_Sheets
-.. _JavaScript: https://en.wikipedia.org/wiki/JavaScript
.. _chompjs: https://github.com/Nykakin/chompjs
.. _curl: https://curl.se/
.. _headless browser: https://en.wikipedia.org/wiki/Headless_browser
.. _js2xml: https://github.com/scrapinghub/js2xml
.. _playwright-python: https://github.com/microsoft/playwright-python
.. _playwright: https://github.com/microsoft/playwright
-.. _pyppeteer: https://pyppeteer.github.io/pyppeteer/
.. _pytesseract: https://github.com/madmaze/pytesseract
.. _scrapy-playwright: https://github.com/scrapy-plugins/scrapy-playwright
.. _tabula-py: https://github.com/chezou/tabula-py
diff --git a/docs/topics/email.rst b/docs/topics/email.rst
deleted file mode 100644
index 1d7bad787..000000000
--- a/docs/topics/email.rst
+++ /dev/null
@@ -1,185 +0,0 @@
-.. _topics-email:
-
-==============
-Sending e-mail
-==============
-
-.. module:: scrapy.mail
- :synopsis: Email sending facility
-
-Although Python makes sending e-mails relatively easy via the :mod:`smtplib`
-library, Scrapy provides its own facility for sending e-mails which is very
-easy to use and it's implemented using :doc:`Twisted non-blocking IO
-`, to avoid interfering with the non-blocking
-IO of the crawler. It also provides a simple API for sending attachments and
-it's very easy to configure, with a few :ref:`settings
-`.
-
-Quick example
-=============
-
-There are two ways to instantiate the mail sender. You can instantiate it using
-the standard ``__init__`` method:
-
-.. code-block:: python
-
- from scrapy.mail import MailSender
-
- mailer = MailSender()
-
-Or you can instantiate it passing a :class:`scrapy.Crawler` instance, which
-will respect the :ref:`settings `:
-
-.. skip: start
-.. code-block:: python
-
- mailer = MailSender.from_crawler(crawler)
-
-And here is how to use it to send an e-mail (without attachments):
-
-.. code-block:: python
-
- mailer.send(
- to=["someone@example.com"],
- subject="Some subject",
- body="Some body",
- cc=["another@example.com"],
- )
-.. skip: end
-
-MailSender class reference
-==========================
-
-The MailSender :ref:`components ` is the preferred class to
-use for sending emails from Scrapy, as it uses :doc:`Twisted non-blocking IO
-`, like the rest of the framework.
-
-.. class:: MailSender(smtphost=None, mailfrom=None, smtpuser=None, smtppass=None, smtpport=None)
-
- :param smtphost: the SMTP host to use for sending the emails. If omitted, the
- :setting:`MAIL_HOST` setting will be used.
- :type smtphost: str
-
- :param mailfrom: the address used to send emails (in the ``From:`` header).
- If omitted, the :setting:`MAIL_FROM` setting will be used.
- :type mailfrom: str
-
- :param smtpuser: the SMTP user. If omitted, the :setting:`MAIL_USER`
- setting will be used. If not given, no SMTP authentication will be
- performed.
- :type smtphost: str or bytes
-
- :param smtppass: the SMTP pass for authentication.
- :type smtppass: str or bytes
-
- :param smtpport: the SMTP port to connect to
- :type smtpport: int
-
- :param smtptls: enforce using SMTP STARTTLS
- :type smtptls: bool
-
- :param smtpssl: enforce using a secure SSL connection
- :type smtpssl: bool
-
- .. method:: send(to, subject, body, cc=None, attachs=(), mimetype='text/plain', charset=None)
-
- Send email to the given recipients.
-
- :param to: the e-mail recipients as a string or as a list of strings
- :type to: str or list
-
- :param subject: the subject of the e-mail
- :type subject: str
-
- :param cc: the e-mails to CC as a string or as a list of strings
- :type cc: str or list
-
- :param body: the e-mail body
- :type body: str
-
- :param attachs: an iterable of tuples ``(attach_name, mimetype,
- file_object)`` where ``attach_name`` is a string with the name that will
- appear on the e-mail's attachment, ``mimetype`` is the mimetype of the
- attachment and ``file_object`` is a readable file object with the
- contents of the attachment
- :type attachs: collections.abc.Iterable
-
- :param mimetype: the MIME type of the e-mail
- :type mimetype: str
-
- :param charset: the character encoding to use for the e-mail contents
- :type charset: str
-
-
-.. _topics-email-settings:
-
-Mail settings
-=============
-
-These settings define the default ``__init__`` method values of the :class:`MailSender`
-class, and can be used to configure e-mail notifications in your project without
-writing any code (for those extensions and code that uses :class:`MailSender`).
-
-.. setting:: MAIL_FROM
-
-MAIL_FROM
----------
-
-Default: ``'scrapy@localhost'``
-
-Sender email to use (``From:`` header) for sending emails.
-
-.. setting:: MAIL_HOST
-
-MAIL_HOST
----------
-
-Default: ``'localhost'``
-
-SMTP host to use for sending emails.
-
-.. setting:: MAIL_PORT
-
-MAIL_PORT
----------
-
-Default: ``25``
-
-SMTP port to use for sending emails.
-
-.. setting:: MAIL_USER
-
-MAIL_USER
----------
-
-Default: ``None``
-
-User to use for SMTP authentication. If disabled no SMTP authentication will be
-performed.
-
-.. setting:: MAIL_PASS
-
-MAIL_PASS
----------
-
-Default: ``None``
-
-Password to use for SMTP authentication, along with :setting:`MAIL_USER`.
-
-.. setting:: MAIL_TLS
-
-MAIL_TLS
---------
-
-Default: ``False``
-
-Enforce using STARTTLS. STARTTLS is a way to take an existing insecure connection, and upgrade it to a secure connection using SSL/TLS.
-
-.. setting:: MAIL_SSL
-
-MAIL_SSL
---------
-
-Default: ``False``
-
-Enforce connecting using an SSL encrypted connection
diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index 0b572ff95..0aab90a43 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -1,117 +1,25 @@
.. _topics-exceptions:
+.. _topics-exceptions-ref:
==========
Exceptions
==========
+Here's a list of all exceptions included in Scrapy and their usage, except for
+the :ref:`download handler exceptions `.
+
.. module:: scrapy.exceptions
- :synopsis: Scrapy exceptions
-.. _topics-exceptions-ref:
+.. autoexception:: CloseSpider
-Built-in Exceptions reference
-=============================
+.. autoexception:: DontCloseSpider
-Here's a list of all exceptions included in Scrapy and their usage.
+.. autoexception:: DropItem
+.. autoexception:: IgnoreRequest
-CloseSpider
------------
+.. autoexception:: NotConfigured
-.. exception:: CloseSpider(reason='cancelled')
+.. autoexception:: NotSupported
- This exception can be raised from a spider callback to request the spider to be
- closed/stopped. Supported arguments:
-
- :param reason: the reason for closing
- :type reason: str
-
-For example:
-
-.. code-block:: python
-
- def parse_page(self, response):
- if "Bandwidth exceeded" in response.body:
- raise CloseSpider("bandwidth_exceeded")
-
-DontCloseSpider
----------------
-
-.. exception:: DontCloseSpider
-
-This exception can be raised in a :signal:`spider_idle` signal handler to
-prevent the spider from being closed.
-
-DropItem
---------
-
-.. exception:: DropItem
-
-The exception that must be raised by item pipeline stages to stop processing an
-Item. For more information see :ref:`topics-item-pipeline`.
-
-IgnoreRequest
--------------
-
-.. exception:: IgnoreRequest
-
-This exception can be raised by the Scheduler or any downloader middleware to
-indicate that the request should be ignored.
-
-NotConfigured
--------------
-
-.. exception:: NotConfigured
-
-This exception can be raised by some components to indicate that they will
-remain disabled. Those components include:
-
-- Extensions
-- Item pipelines
-- Downloader middlewares
-- Spider middlewares
-
-The exception must be raised in the component's ``__init__`` method.
-
-NotSupported
-------------
-
-.. exception:: NotSupported
-
-This exception is raised to indicate an unsupported feature.
-
-StopDownload
--------------
-
-.. versionadded:: 2.2
-
-.. exception:: StopDownload(fail=True)
-
-Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
-signal handler to indicate that no further bytes should be downloaded for a response.
-
-The ``fail`` boolean parameter controls which method will handle the resulting
-response:
-
-* If ``fail=True`` (default), the request errback is called. The response object is
- available as the ``response`` attribute of the ``StopDownload`` exception,
- which is in turn stored as the ``value`` attribute of the received
- :class:`~twisted.python.failure.Failure` object. This means that in an errback
- defined as ``def errback(self, failure)``, the response can be accessed though
- ``failure.value.response``.
-
-* If ``fail=False``, the request callback is called instead.
-
-In both cases, the response could have its body truncated: the body contains
-all bytes received up until the exception is raised, including the bytes
-received in the signal handler that raises the exception. Also, the response
-object is marked with ``"download_stopped"`` in its :attr:`~scrapy.http.Response.flags`
-attribute.
-
-.. note:: ``fail`` is a keyword-only parameter, i.e. raising
- ``StopDownload(False)`` or ``StopDownload(True)`` will raise
- a :class:`TypeError`.
-
-See the documentation for the :class:`~scrapy.signals.bytes_received` and
-:class:`~scrapy.signals.headers_received` signals
-and the :ref:`topics-stop-response-download` topic for additional information and examples.
+.. autoexception:: StopDownload
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 5c078568b..c43b7e20f 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -67,7 +67,7 @@ value of one of their fields:
self.year_to_exporter[year] = (exporter, xml_file)
return self.year_to_exporter[year][0]
- def process_item(self, item, spider):
+ def process_item(self, item):
exporter = self._exporter_for_item(item)
exporter.export_item(item)
return item
@@ -93,33 +93,34 @@ described next.
1. Declaring a serializer in the field
--------------------------------------
-If you use :class:`~scrapy.Item` you can declare a serializer in the
-:ref:`field metadata `. The serializer must be
-a callable which receives a value and returns its serialized form.
+Every :ref:`item type ` except :class:`dict` lets you declare a
+serializer in the :ref:`field metadata `. The serializer
+must be a callable which receives a value and returns its serialized form.
Example:
.. code-block:: python
- import scrapy
+ from dataclasses import dataclass, field
def serialize_price(value):
return f"$ {str(value)}"
- class Product(scrapy.Item):
- name = scrapy.Field()
- price = scrapy.Field(serializer=serialize_price)
+ @dataclass
+ class Product:
+ name: str
+ price: float = field(metadata={"serializer": serialize_price})
2. Overriding the serialize_field() method
------------------------------------------
-You can also override the :meth:`~BaseItemExporter.serialize_field()` method to
+You can also override the :meth:`~BaseItemExporter.serialize_field` method to
customize how your field value will be exported.
-Make sure you call the base class :meth:`~BaseItemExporter.serialize_field()` method
+Make sure you call the base class :meth:`~BaseItemExporter.serialize_field` method
after your custom code.
Example:
@@ -152,7 +153,7 @@ output examples, which assume you're exporting these two items:
BaseItemExporter
----------------
-.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0, dont_fail=False)
+.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding=None, indent=None, dont_fail=False)
This is the (abstract) base class for all Item Exporters. It provides
support for common features used by all (concrete) Item Exporters, such as
@@ -163,9 +164,6 @@ BaseItemExporter
populate their respective instance attributes: :attr:`fields_to_export`,
:attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`.
- .. versionadded:: 2.0
- The *dont_fail* parameter.
-
.. method:: export_item(item)
Exports the given item. This method must be implemented in subclasses.
@@ -213,13 +211,27 @@ BaseItemExporter
- ``None`` (all fields [2]_, default)
- - A list of fields::
+ Fields are exported in declaration order, i.e. the order in which
+ they are defined in the :ref:`item class `. For
+ :class:`dict` items, which have no declared fields, the key order of
+ each item is used instead.
- ['field1', 'field2']
+ .. versionchanged:: VERSION
+ Fields of non-\ :class:`dict` items used to be exported in the
+ order in which they had been populated, except in
+ :class:`CsvItemExporter`, which has always used declaration order.
- - A dict where keys are fields and values are output names::
+ - A list of fields:
- {'field1': 'Field 1', 'field2': 'Field 2'}
+ .. code-block:: python
+
+ ["field1", "field2"]
+
+ - A dict where keys are fields and values are output names:
+
+ .. code-block:: python
+
+ {"field1": "Field 1", "field2": "Field 2"}
.. [1] Not all exporters respect the specified field order.
.. [2] When using :ref:`item objects ` that do not expose
@@ -241,7 +253,7 @@ BaseItemExporter
.. attribute:: indent
- Amount of spaces used to indent the output on each level. Defaults to ``0``.
+ Amount of spaces used to indent the output on each level. Defaults to ``None``.
* ``indent=None`` selects the most compact representation,
all items in the same line with no indentation
@@ -275,7 +287,9 @@ XmlItemExporter
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method.
- A typical output of this exporter would be::
+ A typical output of this exporter would be:
+
+ .. code-block:: xml
@@ -293,11 +307,17 @@ XmlItemExporter
exported by serializing each value inside a ```` element. This is for
convenience, as multi-valued fields are very common.
- For example, the item::
+ For example, the item:
- Item(name=['John', 'Doe'], age='23')
+ .. skip: next
- Would be serialized as::
+ .. code-block:: python
+
+ Item(name=["John", "Doe"], age="23")
+
+ Would be serialized as:
+
+ .. code-block:: xml
@@ -330,7 +350,7 @@ CsvItemExporter
:param join_multivalued: The char (or chars) that will be used for joining
multi-valued fields, if found.
- :type include_headers_line: str
+ :type join_multivalued: str
:param errors: The optional string that specifies how encoding and decoding
errors are to be handled. For more information see
@@ -344,14 +364,14 @@ CsvItemExporter
A typical output of this exporter would be::
- product,price
+ name,price
Color TV,1200
DVD player,200
PickleItemExporter
------------------
-.. class:: PickleItemExporter(file, protocol=0, **kwargs)
+.. class:: PickleItemExporter(file, protocol=4, **kwargs)
Exports items in pickle format to the given file-like object.
@@ -381,10 +401,12 @@ PprintItemExporter
The additional keyword arguments of this ``__init__`` method are passed to the
:class:`BaseItemExporter` ``__init__`` method.
- A typical output of this exporter would be::
+ A typical output of this exporter would be:
- {'name': 'Color TV', 'price': '1200'}
- {'name': 'DVD player', 'price': '200'}
+ .. code-block:: python
+
+ {"name": "Color TV", "price": "1200"}
+ {"name": "DVD player", "price": "200"}
Longer lines (when present) are pretty-formatted.
@@ -402,7 +424,9 @@ JsonItemExporter
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
- A typical output of this exporter would be::
+ A typical output of this exporter would be:
+
+ .. code-block:: json
[{"name": "Color TV", "price": "1200"},
{"name": "DVD player", "price": "200"}]
@@ -431,7 +455,9 @@ JsonLinesItemExporter
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
- A typical output of this exporter would be::
+ A typical output of this exporter would be:
+
+ .. code-block:: json
{"name": "Color TV", "price": "1200"}
{"name": "DVD player", "price": "200"}
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index e1e3dd6b4..78b38cc3f 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -136,6 +136,19 @@ Core Stats extension
Enable the collection of core statistics, provided the stats collection is
enabled (see :ref:`topics-stats`).
+The following stats are collected: :stat:`elapsed_time_seconds`,
+:stat:`finish_reason`, :stat:`finish_time`, :stat:`item_dropped_count`,
+:stat:`item_dropped_reasons_count/{exception}`, :stat:`item_scraped_count`,
+:stat:`response_received_count`, :stat:`start_time`.
+
+Log Count extension
+~~~~~~~~~~~~~~~~~~~
+
+.. module:: scrapy.extensions.logcount
+ :synopsis: Basic stats logging
+
+.. autoclass:: LogCount
+
.. _topics-extensions-ref-telnetconsole:
Telnet console extension
@@ -167,20 +180,16 @@ Memory usage extension
Monitors the memory used by the Scrapy process that runs the spider and:
-1. sends a notification e-mail when it exceeds a certain value
-2. closes the spider when it exceeds a certain value
-
-The notification e-mails can be triggered when a certain warning value is
-reached (:setting:`MEMUSAGE_WARNING_MB`) and when the maximum value is reached
-(:setting:`MEMUSAGE_LIMIT_MB`) which will also cause the spider to be closed
-and the Scrapy process to be terminated.
+1. sends a :signal:`memusage_warning_reached` signal when it exceeds
+ :setting:`MEMUSAGE_WARNING_MB`
+2. closes the spider with the ``"memusage_exceeded"`` reason when it exceeds
+ :setting:`MEMUSAGE_LIMIT_MB`
This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and
can be configured with the following settings:
* :setting:`MEMUSAGE_LIMIT_MB`
* :setting:`MEMUSAGE_WARNING_MB`
-* :setting:`MEMUSAGE_NOTIFY_MAIL`
* :setting:`MEMUSAGE_CHECK_INTERVAL_SECONDS`
Memory debugger extension
@@ -197,7 +206,8 @@ An extension for debugging memory usage. It collects information about:
* objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs`
To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The
-info will be stored in the stats.
+info will be stored in the :stat:`memdebug/gc_garbage_count` and
+:stat:`memdebug/live_refs/{cls}` stats.
.. _topics-extensions-ref-spiderstate:
@@ -243,6 +253,7 @@ settings:
* :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`
* :setting:`CLOSESPIDER_ITEMCOUNT`
* :setting:`CLOSESPIDER_PAGECOUNT`
+* :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM`
* :setting:`CLOSESPIDER_ERRORCOUNT`
.. note::
@@ -256,12 +267,11 @@ settings:
CLOSESPIDER_TIMEOUT
"""""""""""""""""""
-Default: ``0``
+Default: ``0.0``
-An integer which specifies a number of seconds. If the spider remains open for
-more than that number of second, it will be automatically closed with the
-reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by
-timeout.
+If the spider remains open for more than this number of seconds, it will be
+automatically closed with the reason ``closespider_timeout``. If zero (or non
+set), spiders won't be closed by timeout.
.. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM
@@ -324,27 +334,6 @@ closing the spider. If the spider generates more than that number of errors,
it will be closed with the reason ``closespider_errorcount``. If zero (or non
set), spiders won't be closed by number of errors.
-StatsMailer extension
-~~~~~~~~~~~~~~~~~~~~~
-
-.. module:: scrapy.extensions.statsmailer
- :synopsis: StatsMailer extension
-
-.. class:: StatsMailer
-
-This simple extension can be used to send a notification e-mail every time a
-domain has finished scraping, including the Scrapy stats collected. The email
-will be sent to all recipients specified in the :setting:`STATSMAILER_RCPTS`
-setting.
-
-Emails can be sent using the :class:`~scrapy.mail.MailSender` class. To see a
-full list of parameters, including examples on how to instantiate
-:class:`~scrapy.mail.MailSender` and use mail settings, see
-:ref:`topics-email`.
-
-.. module:: scrapy.extensions.debug
- :synopsis: Extensions for debugging Scrapy
-
.. module:: scrapy.extensions.periodic_log
:synopsis: Periodic stats logging
@@ -419,7 +408,7 @@ Example extension configuration:
custom_settings = {
"LOG_LEVEL": "INFO",
"PERIODIC_LOG_STATS": {
- "include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count/"],
+ "include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count"],
},
"PERIODIC_LOG_DELTA": {"include": ["downloader/"]},
"PERIODIC_LOG_TIMING_ENABLED": True,
@@ -464,6 +453,9 @@ Default: ``False``
Debugging extensions
--------------------
+.. module:: scrapy.extensions.debug
+ :synopsis: Extensions for debugging Scrapy
+
Stack trace dump extension
~~~~~~~~~~~~~~~~~~~~~~~~~~
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 2184f2d0e..2f686fd0f 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -92,7 +92,6 @@ Marshal
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
- Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
-
.. _topics-feed-storage:
Storages
@@ -106,14 +105,13 @@ The storages backends supported out of the box are:
- :ref:`topics-feed-storage-fs`
- :ref:`topics-feed-storage-ftp`
-- :ref:`topics-feed-storage-s3` (requires boto3_)
-- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
+- :ref:`topics-feed-storage-s3` (requires the :ref:`s3 ` extra)
+- :ref:`topics-feed-storage-gcs` (requires the :ref:`gcs ` extra)
- :ref:`topics-feed-storage-stdout`
-Some storage backends may be unavailable if the required external libraries are
-not available. For example, the S3 backend is only available if the boto3_
-library is installed.
-
+Some storage backends may be unavailable if the required :ref:`extras `
+are not installed. For example, the S3 backend requires the :ref:`s3 `
+extra.
.. _topics-feed-uri-params:
@@ -143,6 +141,11 @@ Here are some examples to illustrate:
.. note:: :ref:`Spider arguments ` become spider attributes, hence
they can also be used as storage URI parameters.
+.. note:: Only ``%(...)s`` parameters are replaced. Any other percent
+ character is kept as-is, so percent-encoded URIs (e.g. ``%20`` for a
+ space or percent-encoded FTP credentials) and :class:`pathlib.Path`
+ keys containing ``%(...)s`` parameters both work as expected.
+
.. _topics-feed-storage-backends:
@@ -161,7 +164,7 @@ The feeds are stored in the local filesystem.
- Required external libraries: none
Note that for the local filesystem storage (only) you can omit the scheme if
-you specify an absolute path like ``/tmp/export.csv`` (Unix systems only).
+you specify a path (e.g. ``/tmp/export.csv``).
Alternatively you can also use a :class:`pathlib.Path` object.
.. _topics-feed-storage-ftp:
@@ -204,7 +207,7 @@ The feeds are stored on `Amazon S3`_.
- ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
-- Required external libraries: `boto3`_ >= 1.20.0
+- Required extras: :ref:`s3 `
The AWS credentials can be passed as user/password in the URI, or they can be
passed through the following settings:
@@ -215,12 +218,13 @@ passed through the following settings:
.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
-You can also define a custom ACL, custom endpoint, and region name for exported
-feeds using these settings:
+You can also define a custom ACL, custom endpoint, region name and connection
+pool size for exported feeds using these settings:
- :setting:`FEED_STORAGE_S3_ACL`
- :setting:`AWS_ENDPOINT_URL`
- :setting:`AWS_REGION_NAME`
+- :setting:`AWS_MAX_POOL_CONNECTIONS`
The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
storage backend is: ``True``.
@@ -236,8 +240,6 @@ This storage backend uses :ref:`delayed file delivery `.
Google Cloud Storage (GCS)
--------------------------
-.. versionadded:: 2.3
-
The feeds are stored on `Google Cloud Storage`_.
- URI scheme: ``gs``
@@ -246,9 +248,9 @@ The feeds are stored on `Google Cloud Storage`_.
- ``gs://mybucket/path/to/export.csv``
-- Required external libraries: `google-cloud-storage`_.
+- Required extras: :ref:`gcs `
-For more information about authentication, please refer to `Google Cloud documentation `_.
+For more information about authentication, please refer to `Google Cloud documentation `_.
You can set a *Project ID* and *Access Control List (ACL)* through the following settings:
@@ -263,7 +265,6 @@ storage backend is: ``True``.
This storage backend uses :ref:`delayed file delivery `.
-.. _google-cloud-storage: https://cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
.. _topics-feed-storage-stdout:
@@ -303,8 +304,6 @@ feed URI, allowing item delivery to start way before the end of the crawl.
Item filtering
==============
-.. versionadded:: 2.6.0
-
You can filter items that you want to allow for a particular feed by using the
``item_classes`` option in :ref:`feeds options `. Only items of
the specified types will be added to the feed.
@@ -344,8 +343,6 @@ ItemFilter
Post-Processing
===============
-.. versionadded:: 2.6.0
-
Scrapy provides an option to activate plugins to post-process feeds before they are exported
to feed storages. In addition to using :ref:`builtin plugins `, you
can create your own :ref:`plugins `.
@@ -425,8 +422,6 @@ These are the settings used for configuring the feed exports:
FEEDS
-----
-.. versionadded:: 2.1
-
Default: ``{}``
A dictionary in which every key is a feed URI (or a :class:`pathlib.Path`
@@ -437,33 +432,37 @@ This setting is required for enabling the feed export feature.
See :ref:`topics-feed-storage-backends` for supported URI schemes.
-For instance::
+For instance:
+
+.. skip: next
+
+.. code-block:: python
{
- 'items.json': {
- 'format': 'json',
- 'encoding': 'utf8',
- 'store_empty': False,
- 'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'],
- 'fields': None,
- 'indent': 4,
- 'item_export_kwargs': {
- 'export_empty_fields': True,
+ "items.json": {
+ "format": "json",
+ "encoding": "utf8",
+ "store_empty": False,
+ "item_classes": [MyItemClass1, "myproject.items.MyItemClass2"],
+ "fields": None,
+ "indent": 4,
+ "item_export_kwargs": {
+ "export_empty_fields": True,
},
},
- '/home/user/documents/items.xml': {
- 'format': 'xml',
- 'fields': ['name', 'price'],
- 'item_filter': MyCustomFilter1,
- 'encoding': 'latin1',
- 'indent': 8,
+ "/home/user/documents/items.xml": {
+ "format": "xml",
+ "fields": ["name", "price"],
+ "item_filter": MyCustomFilter1,
+ "encoding": "latin1",
+ "indent": 8,
},
- pathlib.Path('items.csv.gz'): {
- 'format': 'csv',
- 'fields': ['price', 'name'],
- 'item_filter': 'myproject.filters.MyCustomFilter2',
- 'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
- 'gzip_compresslevel': 5,
+ pathlib.Path("items.csv.gz"): {
+ "format": "csv",
+ "fields": ["price", "name"],
+ "item_filter": "myproject.filters.MyCustomFilter2",
+ "postprocessing": [MyPlugin1, "scrapy.extensions.postprocessing.GzipPlugin"],
+ "gzip_compresslevel": 5,
},
}
@@ -479,8 +478,6 @@ as a fallback value if that key is not provided for a specific feed definition:
- ``batch_item_count``: falls back to
:setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
- .. versionadded:: 2.3.0
-
- ``encoding``: falls back to :setting:`FEED_EXPORT_ENCODING`.
- ``fields``: falls back to :setting:`FEED_EXPORT_FIELDS`.
@@ -489,20 +486,14 @@ as a fallback value if that key is not provided for a specific feed definition:
If undefined or empty, all items are exported.
- .. versionadded:: 2.6.0
-
- ``item_filter``: a :ref:`filter class ` to filter items to export.
:class:`~scrapy.extensions.feedexport.ItemFilter` is used be default.
- .. versionadded:: 2.6.0
-
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
- ``item_export_kwargs``: :class:`dict` with keyword arguments for the corresponding :ref:`item exporter class `.
- .. versionadded:: 2.4.0
-
- ``overwrite``: whether to overwrite the file if it already exists
(``True``) or append to its content (``False``).
@@ -522,8 +513,6 @@ as a fallback value if that key is not provided for a specific feed definition:
- :ref:`topics-feed-storage-stdout`: ``False`` (overwriting is not supported)
- .. versionadded:: 2.4.0
-
- ``store_empty``: falls back to :setting:`FEED_STORE_EMPTY`.
- ``uri_params``: falls back to :setting:`FEED_URI_PARAMS`.
@@ -532,8 +521,6 @@ as a fallback value if that key is not provided for a specific feed definition:
The plugins will be used in the order of the list passed.
- .. versionadded:: 2.6.0
-
.. setting:: FEED_EXPORT_ENCODING
FEED_EXPORT_ENCODING
@@ -548,10 +535,6 @@ safe numeric encoding (``\uXXXX`` sequences) for historic reasons.
Use ``"utf-8"`` if you want UTF-8 for JSON too.
-.. versionchanged:: 2.8
- The :command:`startproject` command now sets this setting to
- ``"utf-8"`` in the generated ``settings.py`` file.
-
.. setting:: FEED_EXPORT_FIELDS
FEED_EXPORT_FIELDS
@@ -639,6 +622,7 @@ Default:
"file": "scrapy.extensions.feedexport.FileFeedStorage",
"stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
"s3": "scrapy.extensions.feedexport.S3FeedStorage",
+ "gs": "scrapy.extensions.feedexport.GCSFeedStorage",
"ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
}
@@ -700,8 +684,6 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter
FEED_EXPORT_BATCH_ITEM_COUNT
----------------------------
-.. versionadded:: 2.3.0
-
Default: ``0``
If assigned an integer number higher than ``0``, Scrapy generates multiple output files
@@ -771,23 +753,19 @@ The function signature should be as follows:
If :setting:`FEED_EXPORT_BATCH_ITEM_COUNT` is ``0``, ``batch_id``
is always ``1``.
- .. versionadded:: 2.3.0
-
- ``batch_time``: UTC date and time, in ISO format with ``:``
replaced with ``-``.
See :setting:`FEED_EXPORT_BATCH_ITEM_COUNT`.
- .. versionadded:: 2.3.0
-
- ``time``: ``batch_time``, with microseconds set to ``0``.
:type params: dict
:param spider: source spider of the feed items
:type spider: scrapy.Spider
- .. caution:: The function should return a new dictionary, modifying
- the received ``params`` in-place is deprecated.
+ .. caution:: The function must return a new dictionary instead of modifying
+ the received ``params`` in-place.
For example, to include the :attr:`name ` of the
source spider in the feed URI:
@@ -814,6 +792,5 @@ source spider in the feed URI:
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
-.. _boto3: https://github.com/boto/boto3
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
.. _Google Cloud Storage: https://cloud.google.com/storage/
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index dc27ce6ca..951c0f485 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -26,45 +26,39 @@ Writing your own item pipeline
Each item pipeline is a :ref:`component ` that must
implement the following method:
-.. method:: process_item(self, item, spider)
+.. method:: process_item(self, item)
This method is called for every item pipeline component.
`item` is an :ref:`item object `, see
:ref:`supporting-item-types`.
- :meth:`process_item` must either: return an :ref:`item object `,
- return a :class:`~twisted.internet.defer.Deferred` or raise a
- :exc:`~scrapy.exceptions.DropItem` exception.
+ :meth:`process_item` must either return an :ref:`item object `
+ or raise a :exc:`~scrapy.exceptions.DropItem` exception.
Dropped items are no longer processed by further pipeline components.
:param item: the scraped item
:type item: :ref:`item object `
- :param spider: the spider which scraped the item
- :type spider: :class:`~scrapy.Spider` object
-
Additionally, they may also implement the following methods:
-.. method:: open_spider(self, spider)
+.. method:: open_spider(self)
This method is called when the spider is opened.
- :param spider: the spider which was opened
- :type spider: :class:`~scrapy.Spider` object
-
-.. method:: close_spider(self, spider)
+.. method:: close_spider(self)
This method is called when the spider is closed.
- :param spider: the spider which was closed
- :type spider: :class:`~scrapy.Spider` object
+Any of these methods may be defined as a coroutine function (``async def``).
Item pipeline example
=====================
+.. _price-pipeline-example:
+
Price validation and dropping items with no prices
--------------------------------------------------
@@ -82,7 +76,7 @@ contain a price:
class PricePipeline:
vat_factor = 1.15
- def process_item(self, item, spider):
+ def process_item(self, item):
adapter = ItemAdapter(item)
if adapter.get("price"):
if adapter.get("price_excludes_vat"):
@@ -107,13 +101,13 @@ format:
class JsonWriterPipeline:
- def open_spider(self, spider):
+ def open_spider(self):
self.file = open("items.jsonl", "w")
- def close_spider(self, spider):
+ def close_spider(self):
self.file.close()
- def process_item(self, item, spider):
+ def process_item(self, item):
line = json.dumps(ItemAdapter(item).asdict()) + "\n"
self.file.write(line)
return item
@@ -127,7 +121,7 @@ Write items to MongoDB
In this example we'll write items to MongoDB_ using pymongo_.
MongoDB address and database name are specified in Scrapy settings;
-MongoDB collection is named after item class.
+MongoDB collection is specified in a class attribute.
The main point of this example is to show how to :ref:`get the crawler
` and how to clean up the resources properly.
@@ -153,14 +147,14 @@ The main point of this example is to show how to :ref:`get the crawler
mongo_db=crawler.settings.get("MONGO_DATABASE", "items"),
)
- def open_spider(self, spider):
+ def open_spider(self):
self.client = pymongo.MongoClient(self.mongo_uri)
self.db = self.client[self.mongo_db]
- def close_spider(self, spider):
+ def close_spider(self):
self.client.close()
- def process_item(self, item, spider):
+ def process_item(self, item):
self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
return item
@@ -190,7 +184,6 @@ item.
import scrapy
from itemadapter import ItemAdapter
from scrapy.http.request import NO_CALLBACK
- from scrapy.utils.defer import maybe_deferred_to_future
class ScreenshotPipeline:
@@ -199,14 +192,19 @@ item.
SPLASH_URL = "http://localhost:8050/render.png?url={}"
- async def process_item(self, item, spider):
+ def __init__(self, crawler):
+ self.crawler = crawler
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ return cls(crawler)
+
+ async def process_item(self, item):
adapter = ItemAdapter(item)
encoded_item_url = quote(adapter["url"])
screenshot_url = self.SPLASH_URL.format(encoded_item_url)
request = scrapy.Request(screenshot_url, callback=NO_CALLBACK)
- response = await maybe_deferred_to_future(
- spider.crawler.engine.download(request)
- )
+ response = await self.crawler.engine.download_async(request)
if response.status != 200:
# Error happened, return item.
@@ -241,7 +239,7 @@ returns multiples items with the same id:
def __init__(self):
self.ids_seen = set()
- def process_item(self, item, spider):
+ def process_item(self, item):
adapter = ItemAdapter(item)
if adapter["id"] in self.ids_seen:
raise DropItem(f"Item ID already seen: {adapter['id']}")
@@ -250,6 +248,8 @@ returns multiples items with the same id:
return item
+.. _activating-item-pipeline:
+
Activating an Item Pipeline component
=====================================
@@ -266,3 +266,110 @@ To activate an Item Pipeline component you must add its class to the
The integer values you assign to classes in this setting determine the
order in which they run: items go through from lower valued to higher
valued classes. It's customary to define these numbers in the 0-1000 range.
+
+A complete example
+==================
+
+The examples above show item pipeline components on their own. In a project, a
+pipeline is one of four pieces that work together: the :ref:`item
+` your spider produces, the :ref:`spider ` that
+yields it, the pipeline that processes it, and the :setting:`ITEM_PIPELINES`
+setting that enables the pipeline.
+
+The following example wires those pieces together to validate the price of
+books scraped from `books.toscrape.com`_, reusing the ``PricePipeline`` from
+:ref:`price-pipeline-example` above.
+
+Define the item in ``myproject/items.py``:
+
+.. code-block:: python
+
+ from dataclasses import dataclass
+
+
+ @dataclass
+ class BookItem:
+ title: str
+ price: float
+
+Yield instances of that item from your spider, e.g. in
+``myproject/spiders/books.py``:
+
+.. skip: next
+.. code-block:: python
+
+ import scrapy
+
+ from myproject.items import BookItem
+
+
+ class BooksSpider(scrapy.Spider):
+ name = "books"
+ start_urls = ["https://books.toscrape.com/"]
+
+ def parse(self, response):
+ for book in response.css("article.product_pod"):
+ yield BookItem(
+ title=book.css("h3 a::attr(title)").get(),
+ price=float(book.css("p.price_color::text").re_first(r"[\d.]+")),
+ )
+
+Put the ``PricePipeline`` shown earlier in ``myproject/pipelines.py``, and
+enable it in ``myproject/settings.py``:
+
+.. code-block:: python
+
+ ITEM_PIPELINES = {
+ "myproject.pipelines.PricePipeline": 300,
+ }
+
+With these pieces in place, every ``BookItem`` that ``BooksSpider`` yields
+passes through ``PricePipeline`` before it reaches the :ref:`feed exports
+` or any other output.
+
+.. _books.toscrape.com: https://books.toscrape.com/
+
+
+Common pitfalls
+===============
+
+The pipeline does not run
+-------------------------
+
+A pipeline component only runs if its class is listed in the
+:setting:`ITEM_PIPELINES` setting, normally in your project's
+:file:`settings.py` file (see :ref:`activating-item-pipeline`). Adding it to
+the spider or elsewhere has no effect.
+
+To confirm that Scrapy loaded your pipeline, look for a line like this near the
+start of the crawl log::
+
+ [scrapy.middleware] INFO: Enabled item pipelines:
+ ['myproject.pipelines.PricePipeline']
+
+If your pipeline is missing from that list, check that its import path matches
+the :setting:`ITEM_PIPELINES` entry, and that the setting is not being
+overridden, for example by :attr:`~scrapy.Spider.custom_settings` or by a
+redefinition of :setting:`ITEM_PIPELINES` in :file:`settings.py`.
+
+The item is not returned
+------------------------
+
+:meth:`process_item` must return the item (or raise
+:exc:`~scrapy.exceptions.DropItem`). A common mistake is to modify the item but
+forget to return it:
+
+.. code-block:: python
+
+ def process_item(self, item):
+ ItemAdapter(item)["price"] *= 1.15
+ # Bug: returns None, so the next component gets None instead of the item.
+
+Return the item so that the next component, and the rest of Scrapy, can keep
+processing it:
+
+.. code-block:: python
+
+ def process_item(self, item):
+ ItemAdapter(item)["price"] *= 1.15
+ return item
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 0365c95b3..0892dd839 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -23,7 +23,8 @@ Item Types
Scrapy supports the following types of items, via the `itemadapter`_ library:
:ref:`dictionaries `, :ref:`Item objects `,
-:ref:`dataclass objects `, and :ref:`attrs objects `.
+:ref:`dataclass objects