mirror of https://github.com/scrapy/scrapy.git
Compare commits
No commits in common. "master" and "2.17.0" have entirely different histories.
|
|
@ -1,12 +0,0 @@
|
||||||
version: 2
|
|
||||||
updates:
|
|
||||||
- package-ecosystem: github-actions
|
|
||||||
directory: "/"
|
|
||||||
schedule:
|
|
||||||
interval: monthly
|
|
||||||
groups:
|
|
||||||
github-actions:
|
|
||||||
patterns:
|
|
||||||
- "*"
|
|
||||||
cooldown:
|
|
||||||
default-days: 7
|
|
||||||
|
|
@ -1,7 +1,5 @@
|
||||||
name: Auto-close LLM PRs
|
name: Auto-close LLM PRs
|
||||||
# The workflow only reads the pull request body through the API, it never
|
on:
|
||||||
# checks out or runs pull request code, so pull_request_target is safe here.
|
|
||||||
on: # zizmor: ignore[dangerous-triggers]
|
|
||||||
pull_request_target:
|
pull_request_target:
|
||||||
types: [opened]
|
types: [opened]
|
||||||
permissions:
|
permissions:
|
||||||
|
|
@ -13,7 +11,7 @@ jobs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Check PR body and close if LLM-written
|
- name: Check PR body and close if LLM-written
|
||||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
uses: actions/github-script@v6
|
||||||
with:
|
with:
|
||||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
script: |
|
script: |
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,4 @@
|
||||||
name: Checks
|
name: Checks
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
|
|
@ -17,10 +13,6 @@ concurrency:
|
||||||
jobs:
|
jobs:
|
||||||
checks:
|
checks:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
env:
|
|
||||||
# Make uv use the interpreter that actions/setup-python installed instead
|
|
||||||
# of downloading one of its own.
|
|
||||||
UV_PYTHON_PREFERENCE: only-system
|
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
|
|
@ -46,31 +38,21 @@ jobs:
|
||||||
TOXENV: twinecheck
|
TOXENV: twinecheck
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
- uses: actions/checkout@v6
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
|
|
||||||
- name: Set up Python ${{ matrix.python-version }}
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
uses: actions/setup-python@v6
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
|
|
||||||
- name: Set up uv
|
|
||||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
|
||||||
with:
|
|
||||||
cache-dependency-glob: |
|
|
||||||
docs/requirements.txt
|
|
||||||
pyproject.toml
|
|
||||||
tox.ini
|
|
||||||
|
|
||||||
- name: Run check
|
- name: Run check
|
||||||
env: ${{ matrix.env }}
|
env: ${{ matrix.env }}
|
||||||
run: uvx --with tox-uv tox
|
run: |
|
||||||
|
pip install -U tox
|
||||||
|
tox
|
||||||
|
|
||||||
pre-commit:
|
pre-commit:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
- uses: actions/checkout@v6
|
||||||
with:
|
- uses: pre-commit/action@v3.0.1
|
||||||
persist-credentials: false
|
|
||||||
- uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
|
|
||||||
|
|
|
||||||
|
|
@ -1,59 +0,0 @@
|
||||||
---
|
|
||||||
name: codspeed
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches:
|
|
||||||
- master
|
|
||||||
pull_request:
|
|
||||||
paths:
|
|
||||||
- scrapy/**
|
|
||||||
- tests/benchmarks/**
|
|
||||||
- .github/workflows/codspeed.yml
|
|
||||||
- tox.ini
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
concurrency:
|
|
||||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
|
||||||
cancel-in-progress: true
|
|
||||||
|
|
||||||
permissions: {}
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
benchmark:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
env:
|
|
||||||
# Make uv use the interpreter that actions/setup-python installed
|
|
||||||
# instead of downloading one of its own.
|
|
||||||
UV_PYTHON_PREFERENCE: only-system
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
id-token: write # OIDC authentication with CodSpeed
|
|
||||||
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
|
|
||||||
- name: Set up Python 3.14
|
|
||||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
||||||
with:
|
|
||||||
python-version: '3.14'
|
|
||||||
|
|
||||||
- name: Set up uv
|
|
||||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
|
||||||
with:
|
|
||||||
cache-dependency-glob: |
|
|
||||||
pyproject.toml
|
|
||||||
tox.ini
|
|
||||||
|
|
||||||
- name: Install dependencies
|
|
||||||
# tox must stay on PATH for the CodSpeed action to invoke it.
|
|
||||||
run: |
|
|
||||||
uv tool install --with tox-uv tox
|
|
||||||
tox -n -e benchmark
|
|
||||||
- name: Run benchmarks
|
|
||||||
uses: CodSpeedHQ/action@f22792bfac16f3e14eb9fbea76f4a48e9cc22b93 # v4.19.1
|
|
||||||
with:
|
|
||||||
mode: simulation
|
|
||||||
run: tox -e benchmark
|
|
||||||
|
|
@ -1,8 +1,4 @@
|
||||||
name: Publish
|
name: Publish
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
tags:
|
tags:
|
||||||
|
|
@ -13,28 +9,8 @@ concurrency:
|
||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
build:
|
|
||||||
name: Build distribution
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
|
||||||
with:
|
|
||||||
python-version: "3.14"
|
|
||||||
- run: |
|
|
||||||
python -m pip install --upgrade build
|
|
||||||
python -m build
|
|
||||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
||||||
with:
|
|
||||||
name: python-package-distributions
|
|
||||||
path: dist/
|
|
||||||
|
|
||||||
publish:
|
publish:
|
||||||
name: Upload release to PyPI
|
name: Upload release to PyPI
|
||||||
needs:
|
|
||||||
- build
|
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
environment:
|
environment:
|
||||||
name: pypi
|
name: pypi
|
||||||
|
|
@ -42,9 +18,12 @@ jobs:
|
||||||
permissions:
|
permissions:
|
||||||
id-token: write
|
id-token: write
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
- uses: actions/checkout@v6
|
||||||
|
- uses: actions/setup-python@v6
|
||||||
with:
|
with:
|
||||||
name: python-package-distributions
|
python-version: "3.14"
|
||||||
path: dist/
|
- run: |
|
||||||
|
python -m pip install --upgrade build
|
||||||
|
python -m build
|
||||||
- name: Publish to PyPI
|
- name: Publish to PyPI
|
||||||
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
|
uses: pypa/gh-action-pypi-publish@release/v1
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,4 @@
|
||||||
name: macOS
|
name: macOS
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
|
|
@ -18,59 +14,37 @@ jobs:
|
||||||
tests:
|
tests:
|
||||||
runs-on: macos-latest
|
runs-on: macos-latest
|
||||||
env:
|
env:
|
||||||
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
|
PYTEST_ADDOPTS: -n auto
|
||||||
# Make uv use the interpreter that actions/setup-python installed instead
|
|
||||||
# of downloading one of its own.
|
|
||||||
UV_PYTHON_PREFERENCE: only-system
|
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||||
env:
|
env:
|
||||||
- TOXENV: py
|
- TOXENV: py
|
||||||
include:
|
include:
|
||||||
- python-version: '3.14'
|
|
||||||
env:
|
|
||||||
TOXENV: py
|
|
||||||
coverage: true
|
|
||||||
- python-version: '3.14'
|
- python-version: '3.14'
|
||||||
env:
|
env:
|
||||||
TOXENV: no-reactor
|
TOXENV: no-reactor
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
- uses: actions/checkout@v6
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
|
|
||||||
- name: Set up Python ${{ matrix.python-version }}
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
uses: actions/setup-python@v6
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
|
|
||||||
- name: Set up uv
|
|
||||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
|
||||||
with:
|
|
||||||
cache-dependency-glob: |
|
|
||||||
pyproject.toml
|
|
||||||
tox.ini
|
|
||||||
|
|
||||||
- name: Install mitmproxy
|
|
||||||
env:
|
|
||||||
# mitmproxy needs a newer Python than the oldest matrix entries, so let
|
|
||||||
# uv download one where no system interpreter is new enough.
|
|
||||||
UV_PYTHON_PREFERENCE: system
|
|
||||||
run: uv tool install mitmproxy
|
|
||||||
|
|
||||||
- name: Run tests
|
- name: Run tests
|
||||||
env: ${{ matrix.env }}
|
env: ${{ matrix.env }}
|
||||||
run: uvx --with tox-uv tox
|
run: |
|
||||||
|
pip install -U tox
|
||||||
|
tox
|
||||||
|
|
||||||
- name: Upload coverage report
|
- name: Upload coverage report
|
||||||
if: ${{ matrix.coverage }}
|
uses: codecov/codecov-action@v5
|
||||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
|
||||||
|
|
||||||
- name: Upload test results
|
- name: Upload test results
|
||||||
if: ${{ !cancelled() }}
|
if: ${{ !cancelled() }}
|
||||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
uses: codecov/codecov-action@v5
|
||||||
with:
|
with:
|
||||||
report_type: test_results
|
report_type: test_results
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,4 @@
|
||||||
name: Ubuntu
|
name: Ubuntu
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
|
|
@ -18,10 +14,7 @@ jobs:
|
||||||
tests:
|
tests:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
env:
|
env:
|
||||||
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
|
PYTEST_ADDOPTS: -n auto
|
||||||
# Make uv use the interpreter that actions/setup-python installed instead
|
|
||||||
# of downloading one of its own.
|
|
||||||
UV_PYTHON_PREFERENCE: only-system
|
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
|
|
@ -41,15 +34,12 @@ jobs:
|
||||||
- python-version: "3.14"
|
- python-version: "3.14"
|
||||||
env:
|
env:
|
||||||
TOXENV: py
|
TOXENV: py
|
||||||
coverage: true
|
|
||||||
- python-version: "3.14"
|
- python-version: "3.14"
|
||||||
env:
|
env:
|
||||||
TOXENV: default-reactor
|
TOXENV: default-reactor
|
||||||
coverage: true
|
|
||||||
- python-version: "3.14"
|
- python-version: "3.14"
|
||||||
env:
|
env:
|
||||||
TOXENV: no-reactor
|
TOXENV: no-reactor
|
||||||
coverage: true
|
|
||||||
# pinned due to https://github.com/pypy/pypy/issues/5388
|
# pinned due to https://github.com/pypy/pypy/issues/5388
|
||||||
- python-version: pypy3.11-7.3.20
|
- python-version: pypy3.11-7.3.20
|
||||||
env:
|
env:
|
||||||
|
|
@ -59,15 +49,12 @@ jobs:
|
||||||
- python-version: "3.10.19"
|
- python-version: "3.10.19"
|
||||||
env:
|
env:
|
||||||
TOXENV: min
|
TOXENV: min
|
||||||
coverage: true
|
|
||||||
- python-version: "3.10.19"
|
- python-version: "3.10.19"
|
||||||
env:
|
env:
|
||||||
TOXENV: min-default-reactor
|
TOXENV: min-default-reactor
|
||||||
coverage: true
|
|
||||||
- python-version: "3.10.19"
|
- python-version: "3.10.19"
|
||||||
env:
|
env:
|
||||||
TOXENV: min-no-reactor
|
TOXENV: min-no-reactor
|
||||||
coverage: true
|
|
||||||
# pinned due to https://github.com/pypy/pypy/issues/5388
|
# pinned due to https://github.com/pypy/pypy/issues/5388
|
||||||
- python-version: pypy3.11-7.3.20
|
- python-version: pypy3.11-7.3.20
|
||||||
env:
|
env:
|
||||||
|
|
@ -75,20 +62,16 @@ jobs:
|
||||||
- python-version: "3.10.19"
|
- python-version: "3.10.19"
|
||||||
env:
|
env:
|
||||||
TOXENV: min-extra-deps
|
TOXENV: min-extra-deps
|
||||||
coverage: true
|
|
||||||
- python-version: "3.10.19"
|
- python-version: "3.10.19"
|
||||||
env:
|
env:
|
||||||
TOXENV: min-botocore
|
TOXENV: min-botocore
|
||||||
coverage: true
|
|
||||||
|
|
||||||
- python-version: "3.14"
|
- python-version: "3.14"
|
||||||
env:
|
env:
|
||||||
TOXENV: extra-deps
|
TOXENV: extra-deps
|
||||||
coverage: true
|
|
||||||
- python-version: "3.14"
|
- python-version: "3.14"
|
||||||
env:
|
env:
|
||||||
TOXENV: no-reactor-extra-deps
|
TOXENV: no-reactor-extra-deps
|
||||||
coverage: true
|
|
||||||
# pinned due to https://github.com/pypy/pypy/issues/5388
|
# pinned due to https://github.com/pypy/pypy/issues/5388
|
||||||
- python-version: pypy3.11-7.3.20
|
- python-version: pypy3.11-7.3.20
|
||||||
env:
|
env:
|
||||||
|
|
@ -96,15 +79,15 @@ jobs:
|
||||||
- python-version: "3.14"
|
- python-version: "3.14"
|
||||||
env:
|
env:
|
||||||
TOXENV: botocore
|
TOXENV: botocore
|
||||||
coverage: true
|
- python-version: "3.14"
|
||||||
|
env:
|
||||||
|
TOXENV: mitmproxy
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
- uses: actions/checkout@v6
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
|
|
||||||
- name: Set up Python ${{ matrix.python-version }}
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
uses: actions/setup-python@v6
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
|
|
||||||
|
|
@ -114,32 +97,17 @@ jobs:
|
||||||
sudo apt-get update
|
sudo apt-get update
|
||||||
sudo apt-get install libxml2-dev libxslt-dev
|
sudo apt-get install libxml2-dev libxslt-dev
|
||||||
|
|
||||||
- name: Set up uv
|
|
||||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
|
||||||
with:
|
|
||||||
cache-dependency-glob: |
|
|
||||||
pyproject.toml
|
|
||||||
tox.ini
|
|
||||||
|
|
||||||
- name: Install mitmproxy
|
|
||||||
env:
|
|
||||||
# mitmproxy needs a newer Python than the oldest matrix entries, so let
|
|
||||||
# uv download one where no system interpreter is new enough.
|
|
||||||
UV_PYTHON_PREFERENCE: system
|
|
||||||
# mitmproxy has no PyPy wheels, so run it on CPython regardless of the
|
|
||||||
# interpreter under test.
|
|
||||||
run: uv tool install --python cpython mitmproxy
|
|
||||||
|
|
||||||
- name: Run tests
|
- name: Run tests
|
||||||
env: ${{ matrix.env }}
|
env: ${{ matrix.env }}
|
||||||
run: uvx --with tox-uv tox
|
run: |
|
||||||
|
pip install -U tox
|
||||||
|
tox
|
||||||
|
|
||||||
- name: Upload coverage report
|
- name: Upload coverage report
|
||||||
if: ${{ matrix.coverage }}
|
uses: codecov/codecov-action@v5
|
||||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
|
||||||
|
|
||||||
- name: Upload test results
|
- name: Upload test results
|
||||||
if: ${{ !cancelled() }}
|
if: ${{ !cancelled() }}
|
||||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
uses: codecov/codecov-action@v5
|
||||||
with:
|
with:
|
||||||
report_type: test_results
|
report_type: test_results
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,4 @@
|
||||||
name: Windows
|
name: Windows
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
on:
|
on:
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
|
|
@ -18,10 +14,7 @@ jobs:
|
||||||
tests:
|
tests:
|
||||||
runs-on: windows-latest
|
runs-on: windows-latest
|
||||||
env:
|
env:
|
||||||
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
|
PYTEST_ADDOPTS: -n auto
|
||||||
# Make uv use the interpreter that actions/setup-python installed instead
|
|
||||||
# of downloading one of its own.
|
|
||||||
UV_PYTHON_PREFERENCE: only-system
|
|
||||||
strategy:
|
strategy:
|
||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
|
|
@ -41,7 +34,6 @@ jobs:
|
||||||
- python-version: "3.14"
|
- python-version: "3.14"
|
||||||
env:
|
env:
|
||||||
TOXENV: py
|
TOXENV: py
|
||||||
coverage: true
|
|
||||||
- python-version: "3.14"
|
- python-version: "3.14"
|
||||||
env:
|
env:
|
||||||
TOXENV: default-reactor
|
TOXENV: default-reactor
|
||||||
|
|
@ -62,39 +54,24 @@ jobs:
|
||||||
TOXENV: extra-deps
|
TOXENV: extra-deps
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
- uses: actions/checkout@v6
|
||||||
with:
|
|
||||||
persist-credentials: false
|
|
||||||
|
|
||||||
- name: Set up Python ${{ matrix.python-version }}
|
- name: Set up Python ${{ matrix.python-version }}
|
||||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
uses: actions/setup-python@v6
|
||||||
with:
|
with:
|
||||||
python-version: ${{ matrix.python-version }}
|
python-version: ${{ matrix.python-version }}
|
||||||
|
|
||||||
- name: Set up uv
|
|
||||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
|
||||||
with:
|
|
||||||
cache-dependency-glob: |
|
|
||||||
pyproject.toml
|
|
||||||
tox.ini
|
|
||||||
|
|
||||||
- name: Install mitmproxy
|
|
||||||
env:
|
|
||||||
# mitmproxy needs a newer Python than the oldest matrix entries, so let
|
|
||||||
# uv download one where no system interpreter is new enough.
|
|
||||||
UV_PYTHON_PREFERENCE: system
|
|
||||||
run: uv tool install mitmproxy
|
|
||||||
|
|
||||||
- name: Run tests
|
- name: Run tests
|
||||||
env: ${{ matrix.env }}
|
env: ${{ matrix.env }}
|
||||||
run: uvx --with tox-uv tox
|
run: |
|
||||||
|
pip install -U tox
|
||||||
|
tox
|
||||||
|
|
||||||
- name: Upload coverage report
|
- name: Upload coverage report
|
||||||
if: ${{ matrix.coverage }}
|
uses: codecov/codecov-action@v5
|
||||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
|
||||||
|
|
||||||
- name: Upload test results
|
- name: Upload test results
|
||||||
if: ${{ !cancelled() }}
|
if: ${{ !cancelled() }}
|
||||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
uses: codecov/codecov-action@v5
|
||||||
with:
|
with:
|
||||||
report_type: test_results
|
report_type: test_results
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ exclude: |
|
||||||
)
|
)
|
||||||
repos:
|
repos:
|
||||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||||
rev: v0.15.20
|
rev: v0.15.2
|
||||||
hooks:
|
hooks:
|
||||||
- id: ruff-check
|
- id: ruff-check
|
||||||
args: [ --fix ]
|
args: [ --fix ]
|
||||||
|
|
@ -16,7 +16,7 @@ repos:
|
||||||
hooks:
|
hooks:
|
||||||
- id: blacken-docs
|
- id: blacken-docs
|
||||||
additional_dependencies:
|
additional_dependencies:
|
||||||
- black==26.5.1
|
- black==25.9.0
|
||||||
- repo: https://github.com/pre-commit/pre-commit-hooks
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
||||||
rev: v6.0.0
|
rev: v6.0.0
|
||||||
hooks:
|
hooks:
|
||||||
|
|
@ -27,11 +27,6 @@ repos:
|
||||||
hooks:
|
hooks:
|
||||||
- id: sphinx-lint
|
- id: sphinx-lint
|
||||||
- repo: https://github.com/scrapy/sphinx-scrapy
|
- repo: https://github.com/scrapy/sphinx-scrapy
|
||||||
rev: 0.8.10
|
rev: 0.8.8
|
||||||
hooks:
|
hooks:
|
||||||
- id: sphinx-scrapy
|
- id: sphinx-scrapy
|
||||||
- repo: https://github.com/zizmorcore/zizmor-pre-commit
|
|
||||||
rev: v1.28.0
|
|
||||||
hooks:
|
|
||||||
- id: zizmor
|
|
||||||
args: [--no-progress, --fix]
|
|
||||||
|
|
|
||||||
16
README.rst
16
README.rst
|
|
@ -5,7 +5,7 @@
|
||||||
:alt: Scrapy
|
:alt: Scrapy
|
||||||
:width: 480px
|
:width: 480px
|
||||||
|
|
||||||
|version| |python_version| |tests| |coverage| |conda| |deepwiki|
|
|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki|
|
||||||
|
|
||||||
.. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg
|
.. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg
|
||||||
:target: https://pypi.org/pypi/Scrapy
|
:target: https://pypi.org/pypi/Scrapy
|
||||||
|
|
@ -15,9 +15,17 @@
|
||||||
:target: https://pypi.org/pypi/Scrapy
|
:target: https://pypi.org/pypi/Scrapy
|
||||||
:alt: Supported Python Versions
|
:alt: Supported Python Versions
|
||||||
|
|
||||||
.. |tests| image:: https://img.shields.io/github/check-runs/scrapy/scrapy/master?label=tests
|
.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg
|
||||||
:target: https://github.com/scrapy/scrapy/actions?query=branch%3Amaster
|
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
|
||||||
:alt: Tests
|
:alt: Ubuntu
|
||||||
|
|
||||||
|
.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
|
||||||
|
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
|
||||||
|
:alt: macOS
|
||||||
|
|
||||||
|
.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
|
||||||
|
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
|
||||||
|
:alt: Windows
|
||||||
|
|
||||||
.. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg
|
.. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg
|
||||||
:target: https://codecov.io/github/scrapy/scrapy?branch=master
|
:target: https://codecov.io/github/scrapy/scrapy?branch=master
|
||||||
|
|
|
||||||
28
conftest.py
28
conftest.py
|
|
@ -1,7 +1,6 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import os
|
import importlib
|
||||||
from importlib.util import find_spec
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
|
@ -12,7 +11,7 @@ from scrapy.utils.reactor import set_asyncio_event_loop_policy
|
||||||
from scrapy.utils.reactorless import install_reactor_import_hook
|
from scrapy.utils.reactorless import install_reactor_import_hook
|
||||||
from tests.keys import generate_keys
|
from tests.keys import generate_keys
|
||||||
from tests.mockserver.http import MockServer
|
from tests.mockserver.http import MockServer
|
||||||
from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd
|
from tests.mockserver.mitm_proxy import MitmProxy
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import Generator
|
from collections.abc import Generator
|
||||||
|
|
@ -51,12 +50,11 @@ if not H2_ENABLED:
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
if find_spec("httpx2") is None and find_spec("httpx") is None:
|
try:
|
||||||
|
import httpx # noqa: F401
|
||||||
|
except ImportError:
|
||||||
collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
|
collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
|
||||||
|
|
||||||
if find_spec("pytest_codspeed") is None:
|
|
||||||
collect_ignore.append("tests/benchmarks")
|
|
||||||
|
|
||||||
|
|
||||||
def pytest_addoption(parser, pluginmanager):
|
def pytest_addoption(parser, pluginmanager):
|
||||||
if pluginmanager.hasplugin("twisted"):
|
if pluginmanager.hasplugin("twisted"):
|
||||||
|
|
@ -129,16 +127,16 @@ def pytest_runtest_setup(item):
|
||||||
"uvloop",
|
"uvloop",
|
||||||
"botocore",
|
"botocore",
|
||||||
"boto3",
|
"boto3",
|
||||||
|
"mitmproxy",
|
||||||
]
|
]
|
||||||
|
|
||||||
for module in optional_deps:
|
for module in optional_deps:
|
||||||
if item.get_closest_marker(f"requires_{module}") and find_spec(module) is None:
|
if item.get_closest_marker(f"requires_{module}"):
|
||||||
pytest.skip(f"{module} is not installed")
|
try:
|
||||||
|
importlib.import_module(module)
|
||||||
if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None:
|
except ImportError:
|
||||||
pytest.skip("mitmdump is not available")
|
pytest.skip(f"{module} is not installed")
|
||||||
|
|
||||||
|
|
||||||
# Generate localhost certificate files, needed by some tests (but only once if xdist is used)
|
# Generate localhost certificate files, needed by some tests
|
||||||
if "PYTEST_XDIST_WORKER" not in os.environ:
|
generate_keys()
|
||||||
generate_keys()
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,68 @@
|
||||||
|
:orphan:
|
||||||
|
|
||||||
|
======================================
|
||||||
|
Scrapy documentation quick start guide
|
||||||
|
======================================
|
||||||
|
|
||||||
|
This file provides a quick guide on how to compile the Scrapy documentation.
|
||||||
|
|
||||||
|
|
||||||
|
Setup the environment
|
||||||
|
---------------------
|
||||||
|
|
||||||
|
To compile the documentation you need Sphinx Python library. To install it
|
||||||
|
and all its dependencies run the following command from this dir
|
||||||
|
|
||||||
|
::
|
||||||
|
|
||||||
|
pip install -r requirements.txt
|
||||||
|
|
||||||
|
|
||||||
|
Compile the documentation
|
||||||
|
-------------------------
|
||||||
|
|
||||||
|
To compile the documentation (to classic HTML output) run the following command
|
||||||
|
from this dir::
|
||||||
|
|
||||||
|
make html
|
||||||
|
|
||||||
|
Documentation will be generated (in HTML format) inside the ``build/html`` dir.
|
||||||
|
|
||||||
|
|
||||||
|
View the documentation
|
||||||
|
----------------------
|
||||||
|
|
||||||
|
To view the documentation run the following command::
|
||||||
|
|
||||||
|
make htmlview
|
||||||
|
|
||||||
|
This command will fire up your default browser and open the main page of your
|
||||||
|
(previously generated) HTML documentation.
|
||||||
|
|
||||||
|
|
||||||
|
Start over
|
||||||
|
----------
|
||||||
|
|
||||||
|
To clean up all generated documentation files and start from scratch run::
|
||||||
|
|
||||||
|
make clean
|
||||||
|
|
||||||
|
Keep in mind that this command won't touch any documentation source files.
|
||||||
|
|
||||||
|
|
||||||
|
Recreating documentation on the fly
|
||||||
|
-----------------------------------
|
||||||
|
|
||||||
|
There is a way to recreate the doc automatically when you make changes, you
|
||||||
|
need to install watchdog (``pip install watchdog``) and then use::
|
||||||
|
|
||||||
|
make watch
|
||||||
|
|
||||||
|
Alternative method using tox
|
||||||
|
----------------------------
|
||||||
|
|
||||||
|
To compile the documentation to HTML run the following command::
|
||||||
|
|
||||||
|
tox -e docs
|
||||||
|
|
||||||
|
Documentation will be generated inside the ``docs/_build/all`` dir.
|
||||||
|
|
@ -158,7 +158,6 @@ scrapy_intersphinx_enable = [
|
||||||
"itemloaders",
|
"itemloaders",
|
||||||
"parsel",
|
"parsel",
|
||||||
"pytest",
|
"pytest",
|
||||||
"pypug",
|
|
||||||
"scrapy-lint",
|
"scrapy-lint",
|
||||||
"sphinx",
|
"sphinx",
|
||||||
"tox",
|
"tox",
|
||||||
|
|
|
||||||
|
|
@ -323,10 +323,9 @@ deprecation removals are documented in the :ref:`release notes <news>`.
|
||||||
Tests
|
Tests
|
||||||
=====
|
=====
|
||||||
|
|
||||||
Tests are implemented using pytest_. Running tests requires :doc:`tox
|
Tests are implemented using the :doc:`Twisted unit-testing framework
|
||||||
<tox:index>`.
|
<twisted:development/test-standard>`. Running tests requires
|
||||||
|
:doc:`tox <tox:index>`.
|
||||||
.. _pytest: https://pytest.org
|
|
||||||
|
|
||||||
.. _running-tests:
|
.. _running-tests:
|
||||||
|
|
||||||
|
|
@ -372,21 +371,6 @@ To see coverage report install :doc:`coverage <coverage:index>`
|
||||||
|
|
||||||
see output of ``coverage --help`` for more options like html or xml report.
|
see output of ``coverage --help`` for more options like html or xml report.
|
||||||
|
|
||||||
Some tests need a ``mitmdump`` executable (from mitmproxy_) to test against a
|
|
||||||
fully featured proxy server; they are skipped when one cannot be found
|
|
||||||
(``mitmproxy`` is intentionally not a test dependency that would be installed
|
|
||||||
into test venvs, as that sometimes leads to various dependency conflicts).
|
|
||||||
To run these tests, make ``mitmdump`` available in one of these ways:
|
|
||||||
|
|
||||||
* install ``mitmproxy`` so that ``mitmdump`` is on your ``PATH``, e.g. with
|
|
||||||
pipx_ (``pipx install mitmproxy``) or uv_ (``uv tool install mitmproxy``);
|
|
||||||
|
|
||||||
* have uv_ installed, in which case the tests will run
|
|
||||||
``uvx --from mitmproxy mitmdump``;
|
|
||||||
|
|
||||||
* set the ``MITMDUMP`` environment variable to the path of a ``mitmdump``
|
|
||||||
executable.
|
|
||||||
|
|
||||||
Writing tests
|
Writing tests
|
||||||
-------------
|
-------------
|
||||||
|
|
||||||
|
|
@ -414,6 +398,3 @@ And their unit-tests are in::
|
||||||
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
|
.. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
|
||||||
.. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
|
.. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
|
||||||
.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
|
.. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
|
||||||
.. _mitmproxy: https://mitmproxy.org/
|
|
||||||
.. _pipx: https://pipx.pypa.io/
|
|
||||||
.. _uv: https://docs.astral.sh/uv/
|
|
||||||
|
|
|
||||||
22
docs/faq.rst
22
docs/faq.rst
|
|
@ -97,7 +97,7 @@ handler documentation.
|
||||||
How can I scrape an item with attributes in different pages?
|
How can I scrape an item with attributes in different pages?
|
||||||
------------------------------------------------------------
|
------------------------------------------------------------
|
||||||
|
|
||||||
See :ref:`callback-data`.
|
See :ref:`topics-request-response-ref-request-callback-arguments`.
|
||||||
|
|
||||||
How can I simulate a user login in my spider?
|
How can I simulate a user login in my spider?
|
||||||
---------------------------------------------
|
---------------------------------------------
|
||||||
|
|
@ -136,7 +136,7 @@ middleware with a :ref:`custom downloader middleware
|
||||||
<topics-downloader-middleware-custom>` that requires less memory. For example:
|
<topics-downloader-middleware-custom>` that requires less memory. For example:
|
||||||
|
|
||||||
- If your domain names are similar enough, use your own regular expression
|
- If your domain names are similar enough, use your own regular expression
|
||||||
instead of joining the strings in :attr:`~scrapy.Spider.allowed_domains` into
|
instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into
|
||||||
a complex regular expression.
|
a complex regular expression.
|
||||||
|
|
||||||
- If you can meet the installation requirements, use pyre2_ instead of
|
- If you can meet the installation requirements, use pyre2_ instead of
|
||||||
|
|
@ -285,8 +285,7 @@ consume a lot of memory.
|
||||||
In order to avoid parsing all the entire feed at once in memory, you can use
|
In order to avoid parsing all the entire feed at once in memory, you can use
|
||||||
the :func:`~scrapy.utils.iterators.xmliter_lxml` and
|
the :func:`~scrapy.utils.iterators.xmliter_lxml` and
|
||||||
:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what
|
:func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what
|
||||||
:class:`~scrapy.spiders.XMLFeedSpider` and
|
:class:`~scrapy.spiders.XMLFeedSpider` uses.
|
||||||
:class:`~scrapy.spiders.CSVFeedSpider` use.
|
|
||||||
|
|
||||||
.. autofunction:: scrapy.utils.iterators.xmliter_lxml
|
.. autofunction:: scrapy.utils.iterators.xmliter_lxml
|
||||||
|
|
||||||
|
|
@ -332,8 +331,8 @@ section of the site (which varies each time). In that case, the credentials to
|
||||||
log in would be settings, while the url of the section to scrape would be a
|
log in would be settings, while the url of the section to scrape would be a
|
||||||
spider argument.
|
spider argument.
|
||||||
|
|
||||||
I'm scraping an XML document and my XPath selector doesn't return any items
|
I'm scraping a XML document and my XPath selector doesn't return any items
|
||||||
---------------------------------------------------------------------------
|
--------------------------------------------------------------------------
|
||||||
|
|
||||||
You may need to remove namespaces. See :ref:`removing-namespaces`.
|
You may need to remove namespaces. See :ref:`removing-namespaces`.
|
||||||
|
|
||||||
|
|
@ -361,11 +360,10 @@ method for this purpose. For example:
|
||||||
def process_spider_output(self, response, result):
|
def process_spider_output(self, response, result):
|
||||||
for item_or_request in result:
|
for item_or_request in result:
|
||||||
if isinstance(item_or_request, Request):
|
if isinstance(item_or_request, Request):
|
||||||
yield item_or_request
|
|
||||||
continue
|
continue
|
||||||
adapter = ItemAdapter(item_or_request)
|
adapter = ItemAdapter(item)
|
||||||
for _ in range(adapter["multiply_by"]):
|
for _ in range(adapter["multiply_by"]):
|
||||||
yield deepcopy(item_or_request)
|
yield deepcopy(item)
|
||||||
|
|
||||||
Does Scrapy support IPv6 addresses?
|
Does Scrapy support IPv6 addresses?
|
||||||
-----------------------------------
|
-----------------------------------
|
||||||
|
|
@ -384,9 +382,8 @@ How to deal with ``<class 'ValueError'>: filedescriptor out of range in select()
|
||||||
----------------------------------------------------------------------------------------------
|
----------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
This issue `has been reported`_ to appear when running broad crawls in macOS, where the default
|
This issue `has been reported`_ to appear when running broad crawls in macOS, where the default
|
||||||
Twisted reactor was :class:`twisted.internet.selectreactor.SelectReactor` at that time.
|
Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a
|
||||||
If you have switched to this reactor using the :setting:`TWISTED_REACTOR` setting you can switch
|
different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
|
||||||
to a different one in the same way.
|
|
||||||
|
|
||||||
|
|
||||||
.. _faq-stop-response-download:
|
.. _faq-stop-response-download:
|
||||||
|
|
@ -412,6 +409,7 @@ How can I make a blank request?
|
||||||
|
|
||||||
from scrapy import Request
|
from scrapy import Request
|
||||||
|
|
||||||
|
|
||||||
blank_request = Request("data:,")
|
blank_request = Request("data:,")
|
||||||
|
|
||||||
In this case, the URL is set to a data URI scheme. Data URLs allow you to include data
|
In this case, the URL is set to a data URI scheme. Data URLs allow you to include data
|
||||||
|
|
|
||||||
|
|
@ -24,7 +24,7 @@ Having trouble? We'd like to help!
|
||||||
* Ask or search questions in `StackOverflow using the scrapy tag`_.
|
* Ask or search questions in `StackOverflow using the scrapy tag`_.
|
||||||
* Ask or search questions in the `Scrapy subreddit`_.
|
* Ask or search questions in the `Scrapy subreddit`_.
|
||||||
* Search for questions on the archives of the `scrapy-users mailing list`_.
|
* Search for questions on the archives of the `scrapy-users mailing list`_.
|
||||||
* Ask a question in the `#scrapy IRC channel`_.
|
* Ask a question in the `#scrapy IRC channel`_,
|
||||||
* Report bugs with Scrapy in our `issue tracker`_.
|
* Report bugs with Scrapy in our `issue tracker`_.
|
||||||
* Join the Discord community `Scrapy Discord`_.
|
* Join the Discord community `Scrapy Discord`_.
|
||||||
|
|
||||||
|
|
@ -91,15 +91,15 @@ Basic concepts
|
||||||
:doc:`topics/selectors`
|
:doc:`topics/selectors`
|
||||||
Extract the data from web pages using XPath.
|
Extract the data from web pages using XPath.
|
||||||
|
|
||||||
|
:doc:`topics/shell`
|
||||||
|
Test your extraction code in an interactive environment.
|
||||||
|
|
||||||
:doc:`topics/items`
|
:doc:`topics/items`
|
||||||
Define the data you want to scrape.
|
Define the data you want to scrape.
|
||||||
|
|
||||||
:doc:`topics/loaders`
|
:doc:`topics/loaders`
|
||||||
Populate your items with the extracted data.
|
Populate your items with the extracted data.
|
||||||
|
|
||||||
:doc:`topics/shell`
|
|
||||||
Test your extraction code in an interactive environment.
|
|
||||||
|
|
||||||
:doc:`topics/item-pipeline`
|
:doc:`topics/item-pipeline`
|
||||||
Post-process and store your scraped data.
|
Post-process and store your scraped data.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -89,56 +89,6 @@ just like any other Python package.
|
||||||
(See :ref:`platform-specific guides <intro-install-platform-notes>`
|
(See :ref:`platform-specific guides <intro-install-platform-notes>`
|
||||||
below for non-Python dependencies that you may need to install beforehand).
|
below for non-Python dependencies that you may need to install beforehand).
|
||||||
|
|
||||||
.. _extras:
|
|
||||||
|
|
||||||
Optional extras
|
|
||||||
===============
|
|
||||||
|
|
||||||
Scrapy provides optional :ref:`extras <pypug:dependency-specifiers-extras>`
|
|
||||||
that install additional dependencies to enable specific features. To install
|
|
||||||
Scrapy with one or more extras, list them in square brackets:
|
|
||||||
|
|
||||||
.. code-block:: console
|
|
||||||
|
|
||||||
pip install scrapy[s3,images]
|
|
||||||
|
|
||||||
The following extras are available:
|
|
||||||
|
|
||||||
.. list-table::
|
|
||||||
:header-rows: 1
|
|
||||||
|
|
||||||
* - Extra
|
|
||||||
- Provides
|
|
||||||
* - ``bpython``
|
|
||||||
- :ref:`bpython shell <shell-config>`
|
|
||||||
* - ``brotli``
|
|
||||||
- :ref:`Brotli response decompression <http-compression>`
|
|
||||||
* - ``gcs``
|
|
||||||
- :ref:`Google Cloud Storage <topics-feed-storage-gcs>` for
|
|
||||||
:ref:`feed exports <topics-feed-exports>` and
|
|
||||||
:ref:`media pipelines <media-pipeline-gcs>`
|
|
||||||
* - ``httpx``
|
|
||||||
- :ref:`httpx-handler`, including its HTTP/2 and SOCKS proxy support
|
|
||||||
* - ``images``
|
|
||||||
- :ref:`Images pipeline <images-pipeline>`
|
|
||||||
* - ``ipython``
|
|
||||||
- :ref:`IPython shell <shell-config>`
|
|
||||||
* - ``ptpython``
|
|
||||||
- :ref:`ptpython shell <shell-config>`
|
|
||||||
* - ``robotparser``
|
|
||||||
- :ref:`Robotexclusionrulesparser robots.txt parsing <rerp-parser>`
|
|
||||||
* - ``s3``
|
|
||||||
- :ref:`Amazon S3 <topics-feed-storage-s3>` storage for
|
|
||||||
:ref:`feed exports <topics-feed-exports>`,
|
|
||||||
:ref:`media pipelines <media-pipelines-s3>`, and
|
|
||||||
:ref:`S3 downloads <s3-handler>`
|
|
||||||
* - ``twisted-http2``
|
|
||||||
- :ref:`twisted-http2-handler`
|
|
||||||
* - ``uvloop``
|
|
||||||
- `uvloop <https://github.com/MagicStack/uvloop>`_ event loop
|
|
||||||
* - ``zstd``
|
|
||||||
- :ref:`Zstandard response decompression <http-compression>`
|
|
||||||
|
|
||||||
|
|
||||||
.. _intro-install-platform-notes:
|
.. _intro-install-platform-notes:
|
||||||
|
|
||||||
|
|
@ -280,8 +230,8 @@ Installing Scrapy with PyPy on Windows is not tested.
|
||||||
You can check that Scrapy is installed correctly by running ``scrapy bench``.
|
You can check that Scrapy is installed correctly by running ``scrapy bench``.
|
||||||
If this command gives errors such as
|
If this command gives errors such as
|
||||||
``TypeError: ... got 2 unexpected keyword arguments``, this means
|
``TypeError: ... got 2 unexpected keyword arguments``, this means
|
||||||
that the ``PyPyDispatcher`` dependency wasn't installed. To fix this issue, run
|
that setuptools was unable to pick up one PyPy-specific dependency.
|
||||||
``pip install 'PyPyDispatcher>=2.1.0'``.
|
To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
|
||||||
|
|
||||||
|
|
||||||
.. _intro-install-troubleshooting:
|
.. _intro-install-troubleshooting:
|
||||||
|
|
|
||||||
|
|
@ -83,17 +83,16 @@ While this enables you to do very fast crawls (sending multiple concurrent
|
||||||
requests at the same time, in a fault-tolerant way) Scrapy also gives you
|
requests at the same time, in a fault-tolerant way) Scrapy also gives you
|
||||||
control over the politeness of the crawl through :ref:`a few settings
|
control over the politeness of the crawl through :ref:`a few settings
|
||||||
<topics-settings-ref>`. You can do things like setting a download delay between
|
<topics-settings-ref>`. You can do things like setting a download delay between
|
||||||
each request, limiting the amount of concurrent requests per domain, and
|
each request, limiting the amount of concurrent requests per domain or per IP, and
|
||||||
even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries
|
even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries
|
||||||
to figure these settings out automatically.
|
to figure these settings out automatically.
|
||||||
|
|
||||||
.. note::
|
.. note::
|
||||||
|
|
||||||
This is using :ref:`feed exports <topics-feed-exports>` to generate the
|
This is using :ref:`feed exports <topics-feed-exports>` to generate the
|
||||||
JSON Lines file, you can easily change the export format (XML or CSV, for
|
JSON file, you can easily change the export format (XML or CSV, for example) or the
|
||||||
example) or the storage backend (FTP or `Amazon S3`_, for example). You can
|
storage backend (FTP or `Amazon S3`_, for example). You can also write an
|
||||||
also write an :ref:`item pipeline <topics-item-pipeline>` to store the
|
:ref:`item pipeline <topics-item-pipeline>` to store the items in a database.
|
||||||
items in a database.
|
|
||||||
|
|
||||||
|
|
||||||
.. _topics-whatelse:
|
.. _topics-whatelse:
|
||||||
|
|
|
||||||
|
|
@ -769,7 +769,7 @@ crawlers on top of it.
|
||||||
|
|
||||||
Also, a common pattern is to build an item with data from more than one page,
|
Also, a common pattern is to build an item with data from more than one page,
|
||||||
using a :ref:`trick to pass additional data to the callbacks
|
using a :ref:`trick to pass additional data to the callbacks
|
||||||
<callback-data>`.
|
<topics-request-response-ref-request-callback-arguments>`.
|
||||||
|
|
||||||
|
|
||||||
Using spider arguments
|
Using spider arguments
|
||||||
|
|
|
||||||
|
|
@ -3,28 +3,6 @@
|
||||||
Release notes
|
Release notes
|
||||||
=============
|
=============
|
||||||
|
|
||||||
Scrapy VERSION (unreleased)
|
|
||||||
---------------------------
|
|
||||||
|
|
||||||
Backward-incompatible changes
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
- The following runtime usage of zope.interface_ interfaces is removed:
|
|
||||||
|
|
||||||
- :class:`~scrapy.spiderloader.SpiderLoader` and
|
|
||||||
:class:`~scrapy.spiderloader.DummySpiderLoader` are no longer marked
|
|
||||||
as implementing the ``ISpiderLoader`` interface.
|
|
||||||
|
|
||||||
- :func:`~scrapy.spiderloader.get_spider_loader` no longer checks that the
|
|
||||||
configured spider loader implements the ``ISpiderLoader`` interface.
|
|
||||||
|
|
||||||
- :class:`~scrapy.extensions.feedexport.BlockingFeedStorage`,
|
|
||||||
:class:`~scrapy.extensions.feedexport.FileFeedStorage` and
|
|
||||||
:class:`~scrapy.extensions.feedexport.StdoutFeedStorage` are no longer
|
|
||||||
marked as implementing the ``IFeedStorage`` interface.
|
|
||||||
|
|
||||||
(:issue:`6585`, :issue:`7731`)
|
|
||||||
|
|
||||||
.. _release-2.17.0:
|
.. _release-2.17.0:
|
||||||
|
|
||||||
Scrapy 2.17.0 (2026-07-07)
|
Scrapy 2.17.0 (2026-07-07)
|
||||||
|
|
@ -1057,7 +1035,7 @@ Deprecations
|
||||||
New features
|
New features
|
||||||
~~~~~~~~~~~~
|
~~~~~~~~~~~~
|
||||||
|
|
||||||
- Added a new setting, :setting:`REFERRER_POLICIES`, to allow customizing
|
- Added a new setting, :setting:`REFERER_POLICIES`, to allow customizing
|
||||||
supported referrer policies.
|
supported referrer policies.
|
||||||
|
|
||||||
Bug fixes
|
Bug fixes
|
||||||
|
|
@ -2665,7 +2643,7 @@ Deprecation removals
|
||||||
(:issue:`6109`, :issue:`6116`)
|
(:issue:`6109`, :issue:`6116`)
|
||||||
|
|
||||||
- A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that
|
- A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that
|
||||||
does not implement the ``ISpiderLoader`` interface
|
does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface
|
||||||
will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at
|
will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at
|
||||||
run time. Non-compliant classes have been triggering a deprecation warning
|
run time. Non-compliant classes have been triggering a deprecation warning
|
||||||
since Scrapy 1.0.0.
|
since Scrapy 1.0.0.
|
||||||
|
|
@ -5155,7 +5133,7 @@ Bug fixes
|
||||||
* The system file mode creation mask no longer affects the permissions of
|
* The system file mode creation mask no longer affects the permissions of
|
||||||
files generated using the :command:`startproject` command (:issue:`4722`)
|
files generated using the :command:`startproject` command (:issue:`4722`)
|
||||||
|
|
||||||
* ``scrapy.utils.iterators.xmliter`` now supports namespaced node names
|
* :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names
|
||||||
(:issue:`861`, :issue:`4746`)
|
(:issue:`861`, :issue:`4746`)
|
||||||
|
|
||||||
* :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can
|
* :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can
|
||||||
|
|
@ -6068,7 +6046,7 @@ The following changes may impact custom priority queue classes:
|
||||||
|
|
||||||
* A new keyword parameter has been added: ``key``. It is a string
|
* A new keyword parameter has been added: ``key``. It is a string
|
||||||
that is always an empty string for memory queues and indicates the
|
that is always an empty string for memory queues and indicates the
|
||||||
:setting:`JOBDIR` value for disk queues.
|
:setting:`JOB_DIR` value for disk queues.
|
||||||
|
|
||||||
* The parameter for disk queues that contains data from the previous
|
* The parameter for disk queues that contains data from the previous
|
||||||
crawl, ``startprios`` or ``slot_startprios``, is now passed as a
|
crawl, ``startprios`` or ``slot_startprios``, is now passed as a
|
||||||
|
|
@ -6622,7 +6600,7 @@ New features
|
||||||
``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
|
``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
|
||||||
:ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
|
:ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
|
||||||
scheduling improvement on crawls targeting multiple web domains, at the
|
scheduling improvement on crawls targeting multiple web domains, at the
|
||||||
cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`)
|
cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
|
||||||
|
|
||||||
* A new :attr:`.Request.cb_kwargs` attribute
|
* A new :attr:`.Request.cb_kwargs` attribute
|
||||||
provides a cleaner way to pass keyword arguments to callback methods
|
provides a cleaner way to pass keyword arguments to callback methods
|
||||||
|
|
@ -9101,7 +9079,7 @@ New features and settings
|
||||||
- In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`)
|
- In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`)
|
||||||
- Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
|
- Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
|
||||||
- ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
|
- ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
|
||||||
- :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP``
|
- :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP`
|
||||||
- check the documentation for more details
|
- check the documentation for more details
|
||||||
- Added builtin caching DNS resolver (:rev:`2728`)
|
- Added builtin caching DNS resolver (:rev:`2728`)
|
||||||
- Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)
|
- Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)
|
||||||
|
|
|
||||||
|
|
@ -5,4 +5,4 @@ sphinx
|
||||||
sphinx-notfound-page
|
sphinx-notfound-page
|
||||||
sphinx-rtd-theme
|
sphinx-rtd-theme
|
||||||
sphinx-rtd-dark-mode
|
sphinx-rtd-dark-mode
|
||||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10
|
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8
|
||||||
|
|
|
||||||
|
|
@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0
|
||||||
# via
|
# via
|
||||||
# -r docs/requirements.in
|
# -r docs/requirements.in
|
||||||
# sphinx-rtd-dark-mode
|
# sphinx-rtd-dark-mode
|
||||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8
|
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737
|
||||||
# via -r docs/requirements.in
|
# via -r docs/requirements.in
|
||||||
sphinx-sitemap==2.9.0
|
sphinx-sitemap==2.9.0
|
||||||
# via sphinx-scrapy
|
# via sphinx-scrapy
|
||||||
|
|
|
||||||
|
|
@ -21,14 +21,10 @@ The ``ADDONS`` setting is a dict in which every key is an add-on class or its
|
||||||
import path and the value is its priority.
|
import path and the value is its priority.
|
||||||
|
|
||||||
This is an example where two add-ons are enabled in a project's
|
This is an example where two add-ons are enabled in a project's
|
||||||
``settings.py``:
|
``settings.py``::
|
||||||
|
|
||||||
.. skip: next
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
ADDONS = {
|
ADDONS = {
|
||||||
"path.to.someaddon": 0,
|
'path.to.someaddon': 0,
|
||||||
SomeAddonClass: 1,
|
SomeAddonClass: 1,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -60,9 +56,7 @@ the following methods:
|
||||||
:type settings: :class:`~scrapy.settings.BaseSettings`
|
:type settings: :class:`~scrapy.settings.BaseSettings`
|
||||||
|
|
||||||
The settings set by the add-on should use the ``addon`` priority (see
|
The settings set by the add-on should use the ``addon`` priority (see
|
||||||
:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):
|
:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`)::
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
class MyAddon:
|
class MyAddon:
|
||||||
def update_settings(self, settings):
|
def update_settings(self, settings):
|
||||||
|
|
@ -174,6 +168,7 @@ Use a fallback component:
|
||||||
|
|
||||||
from scrapy.utils.misc import build_from_crawler, load_object
|
from scrapy.utils.misc import build_from_crawler, load_object
|
||||||
|
|
||||||
|
|
||||||
FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
|
FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -172,15 +172,46 @@ SpiderLoader API
|
||||||
.. module:: scrapy.spiderloader
|
.. module:: scrapy.spiderloader
|
||||||
:synopsis: The spider loader
|
:synopsis: The spider loader
|
||||||
|
|
||||||
Custom spider loaders can be employed by specifying their path in the
|
.. class:: SpiderLoader
|
||||||
:setting:`SPIDER_LOADER_CLASS` project setting. They must implement
|
|
||||||
:class:`SpiderLoaderProtocol`.
|
|
||||||
|
|
||||||
.. autoclass:: SpiderLoaderProtocol
|
This class is in charge of retrieving and handling the spider classes
|
||||||
:members:
|
defined across the project.
|
||||||
|
|
||||||
.. autoclass:: SpiderLoader
|
Custom spider loaders can be employed by specifying their path in the
|
||||||
:members:
|
:setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement
|
||||||
|
the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an
|
||||||
|
errorless execution.
|
||||||
|
|
||||||
|
.. method:: from_settings(settings)
|
||||||
|
|
||||||
|
This class method is used by Scrapy to create an instance of the class.
|
||||||
|
It's called with the current project settings, and it loads the spiders
|
||||||
|
found recursively in the modules of the :setting:`SPIDER_MODULES`
|
||||||
|
setting.
|
||||||
|
|
||||||
|
:param settings: project settings
|
||||||
|
:type settings: :class:`~scrapy.settings.Settings` instance
|
||||||
|
|
||||||
|
.. method:: load(spider_name)
|
||||||
|
|
||||||
|
Get the Spider class with the given name. It'll look into the previously
|
||||||
|
loaded spiders for a spider class with name ``spider_name`` and will raise
|
||||||
|
a KeyError if not found.
|
||||||
|
|
||||||
|
:param spider_name: spider class name
|
||||||
|
:type spider_name: str
|
||||||
|
|
||||||
|
.. method:: list()
|
||||||
|
|
||||||
|
Get the names of the available spiders in the project.
|
||||||
|
|
||||||
|
.. method:: find_by_request(request)
|
||||||
|
|
||||||
|
List the spiders' names that can handle the given request. Will try to
|
||||||
|
match the request's url against the domains of the spiders.
|
||||||
|
|
||||||
|
:param request: queried request
|
||||||
|
:type request: :class:`~scrapy.Request` instance
|
||||||
|
|
||||||
.. autoclass:: DummySpiderLoader
|
.. autoclass:: DummySpiderLoader
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ asyncio
|
||||||
=======
|
=======
|
||||||
|
|
||||||
Scrapy supports :mod:`asyncio` natively. New projects created with
|
Scrapy supports :mod:`asyncio` natively. New projects created with
|
||||||
:command:`startproject` have asyncio enabled by default, and you can use
|
:command:`scrapy startproject` have asyncio enabled by default, and you can use
|
||||||
:mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine
|
:mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine
|
||||||
<coroutines>`.
|
<coroutines>`.
|
||||||
|
|
||||||
|
|
@ -18,7 +18,7 @@ no additional setup is needed.
|
||||||
Configuring the asyncio reactor
|
Configuring the asyncio reactor
|
||||||
===============================
|
===============================
|
||||||
|
|
||||||
New projects generated with :command:`startproject` have the asyncio
|
New projects generated with :command:`scrapy startproject` have the asyncio
|
||||||
reactor configured by default. No manual setup is needed.
|
reactor configured by default. No manual setup is needed.
|
||||||
|
|
||||||
The :setting:`TWISTED_REACTOR` setting controls which Twisted reactor Scrapy
|
The :setting:`TWISTED_REACTOR` setting controls which Twisted reactor Scrapy
|
||||||
|
|
@ -105,9 +105,6 @@ Scrapy API requires passing a Deferred to it) using the following helpers:
|
||||||
|
|
||||||
.. autofunction:: scrapy.utils.defer.deferred_from_coro
|
.. autofunction:: scrapy.utils.defer.deferred_from_coro
|
||||||
.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f
|
.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f
|
||||||
|
|
||||||
The following function helps with a reverse wrapping:
|
|
||||||
|
|
||||||
.. autofunction:: scrapy.utils.defer.ensure_awaitable
|
.. autofunction:: scrapy.utils.defer.ensure_awaitable
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -150,12 +147,6 @@ Using Scrapy without a Twisted reactor
|
||||||
.. warning::
|
.. warning::
|
||||||
This is currently experimental and may not be suitable for production use.
|
This is currently experimental and may not be suitable for production use.
|
||||||
|
|
||||||
.. note:: As the Twisted download handlers cannot be used without a reactor,
|
|
||||||
the default download handler in this mode is
|
|
||||||
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. You
|
|
||||||
will need to additionally install the :ref:`httpx <extras>` extra to use
|
|
||||||
it, unless you switch to some different handler.
|
|
||||||
|
|
||||||
It's possible to use Scrapy without installing a Twisted reactor at all, by
|
It's possible to use Scrapy without installing a Twisted reactor at all, by
|
||||||
setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this
|
setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this
|
||||||
mode Scrapy will use the asyncio event loop directly, and most of the Scrapy
|
mode Scrapy will use the asyncio event loop directly, and most of the Scrapy
|
||||||
|
|
@ -199,7 +190,7 @@ in future Scrapy versions. The following features are not available:
|
||||||
:class:`~scrapy.crawler.CrawlerProcess`
|
:class:`~scrapy.crawler.CrawlerProcess`
|
||||||
(:class:`~scrapy.crawler.AsyncCrawlerProcess` and
|
(:class:`~scrapy.crawler.AsyncCrawlerProcess` and
|
||||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` are available)
|
:class:`~scrapy.crawler.AsyncCrawlerRunner` are available)
|
||||||
* Twisted-specific DNS resolvers (the :setting:`TWISTED_DNS_RESOLVER` setting)
|
* Twisted-specific DNS resolvers (the :setting:`DNS_RESOLVER` setting)
|
||||||
* User and 3rd-party code that requires a reactor (see :ref:`below
|
* User and 3rd-party code that requires a reactor (see :ref:`below
|
||||||
<asyncio-without-reactor-migrate>` for examples)
|
<asyncio-without-reactor-migrate>` for examples)
|
||||||
|
|
||||||
|
|
@ -227,8 +218,7 @@ for its differences and limitations compared to
|
||||||
|
|
||||||
Additionally, :class:`~scrapy.crawler.AsyncCrawlerProcess` will install a
|
Additionally, :class:`~scrapy.crawler.AsyncCrawlerProcess` will install a
|
||||||
:term:`meta path finder` that prevents :mod:`twisted.internet.reactor` from
|
:term:`meta path finder` that prevents :mod:`twisted.internet.reactor` from
|
||||||
being imported. It will be uninstalled when :meth:`AsyncCrawlerProcess.start()
|
being imported.
|
||||||
<scrapy.crawler.AsyncCrawlerProcess.start>` exits.
|
|
||||||
|
|
||||||
.. _asyncio-without-reactor-migrate:
|
.. _asyncio-without-reactor-migrate:
|
||||||
|
|
||||||
|
|
@ -267,7 +257,6 @@ Here are some examples of APIs and patterns that need a replacement:
|
||||||
|
|
||||||
Scrapy provides unified helpers for some of these examples:
|
Scrapy provides unified helpers for some of these examples:
|
||||||
|
|
||||||
.. autofunction:: scrapy.utils.asyncio.sleep
|
|
||||||
.. autofunction:: scrapy.utils.asyncio.call_later
|
.. autofunction:: scrapy.utils.asyncio.call_later
|
||||||
.. autofunction:: scrapy.utils.asyncio.create_looping_call
|
.. autofunction:: scrapy.utils.asyncio.create_looping_call
|
||||||
.. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall
|
.. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall
|
||||||
|
|
@ -326,7 +315,8 @@ implementations, :class:`~asyncio.ProactorEventLoop` (default) and
|
||||||
:class:`~asyncio.SelectorEventLoop` works with Twisted.
|
:class:`~asyncio.SelectorEventLoop` works with Twisted.
|
||||||
|
|
||||||
Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop`
|
Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop`
|
||||||
automatically when installing the asyncio reactor.
|
automatically when you change the :setting:`TWISTED_REACTOR` setting or call
|
||||||
|
:func:`~scrapy.utils.reactor.install_reactor`.
|
||||||
|
|
||||||
.. note:: Other libraries you use may require
|
.. note:: Other libraries you use may require
|
||||||
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
|
:class:`~asyncio.ProactorEventLoop`, e.g. because it supports
|
||||||
|
|
|
||||||
|
|
@ -75,7 +75,7 @@ AutoThrottle algorithm adjusts download delays based on the following rules:
|
||||||
.. _download-latency:
|
.. _download-latency:
|
||||||
|
|
||||||
In Scrapy, the download latency is measured as the time elapsed between
|
In Scrapy, the download latency is measured as the time elapsed between
|
||||||
sending the request and receiving the HTTP headers.
|
establishing the TCP connection and receiving the HTTP headers.
|
||||||
|
|
||||||
Note that these latencies are very hard to measure accurately in a cooperative
|
Note that these latencies are very hard to measure accurately in a cooperative
|
||||||
multitasking environment because Scrapy may be busy processing a spider
|
multitasking environment because Scrapy may be busy processing a spider
|
||||||
|
|
|
||||||
|
|
@ -114,8 +114,8 @@ some usage help and the available commands::
|
||||||
scrapy <command> [options] [args]
|
scrapy <command> [options] [args]
|
||||||
|
|
||||||
Available commands:
|
Available commands:
|
||||||
|
crawl Run a spider
|
||||||
fetch Fetch a URL using the Scrapy downloader
|
fetch Fetch a URL using the Scrapy downloader
|
||||||
runspider Run a spider from a Python file, no project required
|
|
||||||
[...]
|
[...]
|
||||||
|
|
||||||
The first line will print the currently active project if you're inside a
|
The first line will print the currently active project if you're inside a
|
||||||
|
|
@ -199,7 +199,6 @@ Global commands:
|
||||||
* :command:`fetch`
|
* :command:`fetch`
|
||||||
* :command:`view`
|
* :command:`view`
|
||||||
* :command:`version`
|
* :command:`version`
|
||||||
* :command:`bench`
|
|
||||||
|
|
||||||
Project-only commands:
|
Project-only commands:
|
||||||
|
|
||||||
|
|
@ -208,6 +207,7 @@ Project-only commands:
|
||||||
* :command:`list`
|
* :command:`list`
|
||||||
* :command:`edit`
|
* :command:`edit`
|
||||||
* :command:`parse`
|
* :command:`parse`
|
||||||
|
* :command:`bench`
|
||||||
|
|
||||||
.. command:: startproject
|
.. command:: startproject
|
||||||
|
|
||||||
|
|
@ -263,9 +263,7 @@ crawl
|
||||||
* Syntax: ``scrapy crawl <spider>``
|
* Syntax: ``scrapy crawl <spider>``
|
||||||
* Requires project: *yes*
|
* Requires project: *yes*
|
||||||
|
|
||||||
Start crawling using the spider with the given :attr:`~scrapy.Spider.name`,
|
Start crawling using a spider.
|
||||||
which must be one of those that :command:`list` reports. To run a spider from a
|
|
||||||
file instead, use :command:`runspider`.
|
|
||||||
|
|
||||||
Supported options:
|
Supported options:
|
||||||
|
|
||||||
|
|
@ -311,25 +309,11 @@ Usage examples::
|
||||||
* parse_item
|
* parse_item
|
||||||
|
|
||||||
$ scrapy check
|
$ scrapy check
|
||||||
F.F.
|
[FAILED] first_spider:parse_item
|
||||||
======================================================================
|
>>> 'RetailPricex' field is missing
|
||||||
FAIL: [first_spider] parse (@returns post-hook)
|
|
||||||
----------------------------------------------------------------------
|
|
||||||
Traceback (most recent call last):
|
|
||||||
...
|
|
||||||
scrapy.exceptions.ContractFail: Returned 92 requests, expected 0..4
|
|
||||||
|
|
||||||
======================================================================
|
[FAILED] first_spider:parse
|
||||||
FAIL: [first_spider] parse_item (@scrapes post-hook)
|
>>> Returned 92 requests, expected 0..4
|
||||||
----------------------------------------------------------------------
|
|
||||||
Traceback (most recent call last):
|
|
||||||
...
|
|
||||||
scrapy.exceptions.ContractFail: Missing fields: RetailPricex
|
|
||||||
|
|
||||||
----------------------------------------------------------------------
|
|
||||||
Ran 4 contracts in 0.174s
|
|
||||||
|
|
||||||
FAILED (failures=2)
|
|
||||||
|
|
||||||
.. skip: end
|
.. skip: end
|
||||||
|
|
||||||
|
|
@ -393,7 +377,7 @@ Supported options:
|
||||||
|
|
||||||
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
|
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
|
||||||
|
|
||||||
* ``--headers``: print the request's and response's HTTP headers instead of the response's body
|
* ``--headers``: print the response's HTTP headers instead of the response's body
|
||||||
|
|
||||||
* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
|
* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
|
||||||
|
|
||||||
|
|
@ -403,19 +387,15 @@ Usage examples::
|
||||||
[ ... html content here ... ]
|
[ ... html content here ... ]
|
||||||
|
|
||||||
$ scrapy fetch --nolog --headers http://www.example.com/
|
$ scrapy fetch --nolog --headers http://www.example.com/
|
||||||
> Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
|
{'Accept-Ranges': ['bytes'],
|
||||||
> Accept-Language: en
|
'Age': ['1263 '],
|
||||||
> User-Agent: Scrapy/2.16.0 (+https://scrapy.org)
|
'Connection': ['close '],
|
||||||
> Accept-Encoding: gzip, deflate, br
|
'Content-Length': ['596'],
|
||||||
>
|
'Content-Type': ['text/html; charset=UTF-8'],
|
||||||
< Date: Wed, 08 Jul 2026 06:15:01 GMT
|
'Date': ['Wed, 18 Aug 2010 23:59:46 GMT'],
|
||||||
< Content-Type: text/html
|
'Etag': ['"573c1-254-48c9c87349680"'],
|
||||||
< Server: cloudflare
|
'Last-Modified': ['Fri, 30 Jul 2010 15:30:18 GMT'],
|
||||||
< Last-Modified: Wed, 01 Jul 2026 17:50:18 GMT
|
'Server': ['Apache/2.2.3 (CentOS)']}
|
||||||
< Allow: GET, HEAD
|
|
||||||
< Cf-Cache-Status: HIT
|
|
||||||
< Age: 8184
|
|
||||||
< Cf-Ray: a17cf3b80eddf141-DME
|
|
||||||
|
|
||||||
.. command:: view
|
.. command:: view
|
||||||
|
|
||||||
|
|
@ -496,7 +476,7 @@ Supported options:
|
||||||
|
|
||||||
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
|
* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
|
||||||
|
|
||||||
* ``-a NAME=VALUE``: set spider argument (may be repeated)
|
* ``--a NAME=VALUE``: set spider argument (may be repeated)
|
||||||
|
|
||||||
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
|
* ``--callback`` or ``-c``: spider method to use as callback for parsing the
|
||||||
response
|
response
|
||||||
|
|
@ -573,9 +553,8 @@ runspider
|
||||||
* Syntax: ``scrapy runspider <spider_file.py>``
|
* Syntax: ``scrapy runspider <spider_file.py>``
|
||||||
* Requires project: *no*
|
* Requires project: *no*
|
||||||
|
|
||||||
Run the spider defined in the given Python file, without requiring a project.
|
Run a spider self-contained in a Python file, without having to create a
|
||||||
|
project.
|
||||||
Supported options: the same as :command:`crawl`.
|
|
||||||
|
|
||||||
Example usage::
|
Example usage::
|
||||||
|
|
||||||
|
|
@ -626,10 +605,7 @@ shouldn't matter to the user running the command, but when the user :ref:`needs
|
||||||
a non-default Twisted reactor <disable-asyncio>`, it may be important.
|
a non-default Twisted reactor <disable-asyncio>`, it may be important.
|
||||||
|
|
||||||
Scrapy decides which of these two classes to use based on the value of the
|
Scrapy decides which of these two classes to use based on the value of the
|
||||||
:setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED` settings.
|
:setting:`TWISTED_REACTOR` setting. If the setting value is the default one
|
||||||
With :setting:`TWISTED_REACTOR_ENABLED` set to ``False`` it will use
|
|
||||||
:class:`~scrapy.crawler.AsyncCrawlerProcess`. Otherwise, if the
|
|
||||||
:setting:`TWISTED_REACTOR` value is the default one
|
|
||||||
(``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``),
|
(``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``),
|
||||||
:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used, otherwise
|
:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used, otherwise
|
||||||
:class:`~scrapy.crawler.CrawlerProcess` will be used. The :ref:`spider settings
|
:class:`~scrapy.crawler.CrawlerProcess` will be used. The :ref:`spider settings
|
||||||
|
|
@ -668,8 +644,6 @@ Example:
|
||||||
|
|
||||||
COMMANDS_MODULE = "mybot.commands"
|
COMMANDS_MODULE = "mybot.commands"
|
||||||
|
|
||||||
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
|
|
||||||
|
|
||||||
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
|
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
|
||||||
|
|
||||||
Register commands via setup.py entry points
|
Register commands via setup.py entry points
|
||||||
|
|
|
||||||
|
|
@ -30,15 +30,43 @@ You can use the following contracts:
|
||||||
|
|
||||||
.. module:: scrapy.contracts.default
|
.. module:: scrapy.contracts.default
|
||||||
|
|
||||||
.. autoclass:: UrlContract
|
.. class:: UrlContract
|
||||||
|
|
||||||
.. autoclass:: CallbackKeywordArgumentsContract
|
This contract (``@url``) sets the sample URL used when checking other
|
||||||
|
contract conditions for this spider. This contract is mandatory. All
|
||||||
|
callbacks lacking this contract are ignored when running the checks::
|
||||||
|
|
||||||
.. autoclass:: MetadataContract
|
@url url
|
||||||
|
|
||||||
.. autoclass:: ReturnsContract
|
.. class:: CallbackKeywordArgumentsContract
|
||||||
|
|
||||||
.. autoclass:: ScrapesContract
|
This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`
|
||||||
|
attribute for the sample request. It must be a valid JSON dictionary.
|
||||||
|
::
|
||||||
|
|
||||||
|
@cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
|
||||||
|
|
||||||
|
.. class:: MetadataContract
|
||||||
|
|
||||||
|
This contract (``@meta``) sets the :attr:`meta <scrapy.Request.meta>`
|
||||||
|
attribute for the sample request. It must be a valid JSON dictionary.
|
||||||
|
::
|
||||||
|
|
||||||
|
@meta {"arg1": "value1", "arg2": "value2", ...}
|
||||||
|
|
||||||
|
.. class:: ReturnsContract
|
||||||
|
|
||||||
|
This contract (``@returns``) sets lower and upper bounds for the items and
|
||||||
|
requests returned by the spider. The upper bound is optional::
|
||||||
|
|
||||||
|
@returns item(s)|request(s) [min [max]]
|
||||||
|
|
||||||
|
.. class:: ScrapesContract
|
||||||
|
|
||||||
|
This contract (``@scrapes``) checks that all the items returned by the
|
||||||
|
callback have the specified fields::
|
||||||
|
|
||||||
|
@scrapes field_1 field_2 ...
|
||||||
|
|
||||||
Use the :command:`check` command to run the contract checks.
|
Use the :command:`check` command to run the contract checks.
|
||||||
|
|
||||||
|
|
@ -61,16 +89,30 @@ override three methods:
|
||||||
|
|
||||||
.. module:: scrapy.contracts
|
.. module:: scrapy.contracts
|
||||||
|
|
||||||
.. autoclass:: Contract
|
.. class:: Contract(method, *args)
|
||||||
|
|
||||||
.. automethod:: adjust_request_args
|
:param method: callback function to which the contract is associated
|
||||||
|
:type method: collections.abc.Callable
|
||||||
|
|
||||||
.. method:: pre_process(response)
|
:param args: list of arguments passed into the docstring (whitespace
|
||||||
|
separated)
|
||||||
|
:type args: list
|
||||||
|
|
||||||
|
.. method:: Contract.adjust_request_args(args)
|
||||||
|
|
||||||
|
This receives a ``dict`` as an argument containing default arguments
|
||||||
|
for request object. :class:`~scrapy.Request` is used by default,
|
||||||
|
but this can be changed with the ``request_cls`` attribute.
|
||||||
|
If multiple contracts in chain have this attribute defined, the last one is used.
|
||||||
|
|
||||||
|
Must return the same or a modified version of it.
|
||||||
|
|
||||||
|
.. method:: Contract.pre_process(response)
|
||||||
|
|
||||||
This allows hooking in various checks on the response received from the
|
This allows hooking in various checks on the response received from the
|
||||||
sample request, before it's being passed to the callback.
|
sample request, before it's being passed to the callback.
|
||||||
|
|
||||||
.. method:: post_process(output)
|
.. method:: Contract.post_process(output)
|
||||||
|
|
||||||
This allows processing the output of the callback. Iterators are
|
This allows processing the output of the callback. Iterators are
|
||||||
converted to lists before being passed to this hook.
|
converted to lists before being passed to this hook.
|
||||||
|
|
|
||||||
|
|
@ -16,14 +16,12 @@ Supported callables
|
||||||
The following callables may be defined as coroutines using ``async def``, and
|
The following callables may be defined as coroutines using ``async def``, and
|
||||||
hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
|
hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
|
||||||
|
|
||||||
- The :meth:`~scrapy.Spider.start` spider method, which *must* be
|
- The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be
|
||||||
defined as an :term:`asynchronous generator`.
|
defined as an :term:`asynchronous generator`.
|
||||||
|
|
||||||
.. versionadded:: 2.13
|
.. versionadded:: 2.13
|
||||||
|
|
||||||
- :class:`~scrapy.Request` :ref:`callbacks <callbacks>`, which may
|
- :class:`~scrapy.Request` callbacks.
|
||||||
also be defined as :term:`asynchronous generators <asynchronous
|
|
||||||
generator>`.
|
|
||||||
|
|
||||||
- The :meth:`process_item` method of
|
- The :meth:`process_item` method of
|
||||||
:ref:`item pipelines <topics-item-pipeline>`.
|
:ref:`item pipelines <topics-item-pipeline>`.
|
||||||
|
|
@ -206,15 +204,13 @@ This means you can use many useful Python libraries providing such code:
|
||||||
Common use cases for asynchronous code include:
|
Common use cases for asynchronous code include:
|
||||||
|
|
||||||
* requesting data from websites, databases and other services (in
|
* requesting data from websites, databases and other services (in
|
||||||
:meth:`~scrapy.Spider.start`, callbacks, pipelines and
|
:meth:`~scrapy.spiders.Spider.start`, callbacks, pipelines and
|
||||||
middlewares);
|
middlewares);
|
||||||
* storing data in databases (in pipelines and middlewares);
|
* storing data in databases (in pipelines and middlewares);
|
||||||
* delaying the spider initialization until some external event (in the
|
* delaying the spider initialization until some external event (in the
|
||||||
:signal:`spider_opened` handler);
|
:signal:`spider_opened` handler);
|
||||||
* calling asynchronous Scrapy methods like
|
* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download`
|
||||||
:meth:`ExecutionEngine.download_async()
|
(see :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
|
||||||
<scrapy.core.engine.ExecutionEngine.download_async>` (see :ref:`the
|
|
||||||
screenshot pipeline example <ScreenshotPipeline>`).
|
|
||||||
|
|
||||||
.. _aio-libs: https://github.com/aio-libs
|
.. _aio-libs: https://github.com/aio-libs
|
||||||
|
|
||||||
|
|
@ -272,5 +268,5 @@ You can also send multiple requests in parallel:
|
||||||
yield {
|
yield {
|
||||||
"h1": response.css("h1::text").get(),
|
"h1": response.css("h1::text").get(),
|
||||||
"price": responses[0].css(".price::text").get(),
|
"price": responses[0].css(".price::text").get(),
|
||||||
"color": responses[1].css(".color::text").get(),
|
"price2": responses[1].css(".color::text").get(),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -246,6 +246,7 @@ also request each page to get every quote on the site:
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
import scrapy
|
import scrapy
|
||||||
|
import json
|
||||||
|
|
||||||
|
|
||||||
class QuoteSpider(scrapy.Spider):
|
class QuoteSpider(scrapy.Spider):
|
||||||
|
|
@ -255,7 +256,7 @@ also request each page to get every quote on the site:
|
||||||
start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"]
|
start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"]
|
||||||
|
|
||||||
def parse(self, response):
|
def parse(self, response):
|
||||||
data = response.json()
|
data = json.loads(response.text)
|
||||||
for quote in data["quotes"]:
|
for quote in data["quotes"]:
|
||||||
yield {"quote": quote["text"]}
|
yield {"quote": quote["text"]}
|
||||||
if data["has_next"]:
|
if data["has_next"]:
|
||||||
|
|
|
||||||
|
|
@ -89,7 +89,7 @@ the following API:
|
||||||
If ``True``, the handler will only be instantiated when the first
|
If ``True``, the handler will only be instantiated when the first
|
||||||
request handled by it needs to be downloaded.
|
request handled by it needs to be downloaded.
|
||||||
|
|
||||||
.. method:: download_request(request: Request) -> Response
|
.. method:: download_request(request: Request) -> Response:
|
||||||
:async:
|
:async:
|
||||||
|
|
||||||
Download the given request and return a response.
|
Download the given request and return a response.
|
||||||
|
|
@ -173,8 +173,6 @@ of this package for more information.
|
||||||
H2DownloadHandler
|
H2DownloadHandler
|
||||||
-----------------
|
-----------------
|
||||||
|
|
||||||
.. note:: Requires the :ref:`twisted-http2 <extras>` extra.
|
|
||||||
|
|
||||||
.. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler
|
.. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler
|
||||||
|
|
||||||
| Supported scheme: ``https``.
|
| Supported scheme: ``https``.
|
||||||
|
|
@ -187,6 +185,9 @@ for them.
|
||||||
|
|
||||||
It's implemented using :mod:`twisted.web.client` and the ``h2`` library.
|
It's implemented using :mod:`twisted.web.client` and the ``h2`` library.
|
||||||
|
|
||||||
|
For this handler to work you need to install the ``Twisted[http2]`` extra
|
||||||
|
dependency.
|
||||||
|
|
||||||
If you want to use this handler you need to replace the default one for the
|
If you want to use this handler you need to replace the default one for the
|
||||||
``https`` scheme:
|
``https`` scheme:
|
||||||
|
|
||||||
|
|
@ -272,13 +273,9 @@ Other limitations:
|
||||||
|
|
||||||
- HTTPS proxies to HTTPS destinations are not supported.
|
- HTTPS proxies to HTTPS destinations are not supported.
|
||||||
|
|
||||||
.. _httpx-handler:
|
|
||||||
|
|
||||||
HttpxDownloadHandler
|
HttpxDownloadHandler
|
||||||
--------------------
|
--------------------
|
||||||
|
|
||||||
.. note:: Requires the :ref:`httpx <extras>` extra.
|
|
||||||
|
|
||||||
.. versionadded:: 2.15.0
|
.. versionadded:: 2.15.0
|
||||||
|
|
||||||
.. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler
|
.. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler
|
||||||
|
|
@ -291,9 +288,7 @@ HttpxDownloadHandler
|
||||||
This handler supports ``http://host/path`` and ``https://host/path`` URLs and
|
This handler supports ``http://host/path`` and ``https://host/path`` URLs and
|
||||||
uses the HTTP/1.1 or HTTP/2 protocol for them.
|
uses the HTTP/1.1 or HTTP/2 protocol for them.
|
||||||
|
|
||||||
It's implemented using the httpx2_ library.
|
It's implemented using the ``httpx`` library and needs it to be installed.
|
||||||
|
|
||||||
.. _httpx2: https://httpx2.pydantic.dev/
|
|
||||||
|
|
||||||
If you want to use this handler you need to replace the default ones for the
|
If you want to use this handler you need to replace the default ones for the
|
||||||
``http`` and ``https`` schemes:
|
``http`` and ``https`` schemes:
|
||||||
|
|
@ -316,8 +311,8 @@ Features and limitations
|
||||||
|
|
||||||
=========================== =======================================
|
=========================== =======================================
|
||||||
HTTP proxies Yes
|
HTTP proxies Yes
|
||||||
SOCKS proxies Yes (SOCKS5)
|
SOCKS proxies Yes (SOCKS5; requires ``httpx[socks]``)
|
||||||
HTTP/2 Yes
|
HTTP/2 Yes (requires ``httpx[http2]``)
|
||||||
``response.certificate`` DER bytes
|
``response.certificate`` DER bytes
|
||||||
Per-request ``bindaddress`` No (not supported by the library)
|
Per-request ``bindaddress`` No (not supported by the library)
|
||||||
TLS implementation Standard library ``ssl``
|
TLS implementation Standard library ``ssl``
|
||||||
|
|
@ -338,7 +333,8 @@ HTTPX_HTTP2_ENABLED
|
||||||
|
|
||||||
Default: ``False``
|
Default: ``False``
|
||||||
|
|
||||||
Whether to enable HTTP/2 support in this handler.
|
Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra
|
||||||
|
needs to be installed if you want to enable this setting.
|
||||||
|
|
||||||
Built-in non-HTTP download handlers reference
|
Built-in non-HTTP download handlers reference
|
||||||
=============================================
|
=============================================
|
||||||
|
|
@ -382,13 +378,9 @@ This handler supports ``ftp://host/path`` FTP URIs.
|
||||||
|
|
||||||
It's implemented using :mod:`twisted.protocols.ftp`.
|
It's implemented using :mod:`twisted.protocols.ftp`.
|
||||||
|
|
||||||
.. _s3-handler:
|
|
||||||
|
|
||||||
S3DownloadHandler
|
S3DownloadHandler
|
||||||
-----------------
|
-----------------
|
||||||
|
|
||||||
.. note:: Requires the :ref:`s3 <extras>` extra.
|
|
||||||
|
|
||||||
.. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler
|
.. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler
|
||||||
|
|
||||||
| Supported scheme: ``s3``.
|
| Supported scheme: ``s3``.
|
||||||
|
|
@ -398,6 +390,4 @@ S3DownloadHandler
|
||||||
|
|
||||||
This handler supports ``s3://bucket/path`` S3 URIs.
|
This handler supports ``s3://bucket/path`` S3 URIs.
|
||||||
|
|
||||||
It's implemented using the botocore_ library.
|
It's implemented using the ``botocore`` library and needs it to be installed.
|
||||||
|
|
||||||
.. _botocore: https://github.com/boto/botocore
|
|
||||||
|
|
|
||||||
|
|
@ -505,7 +505,7 @@ Filesystem storage backend (default)
|
||||||
|
|
||||||
* ``response_body`` - the plain response body
|
* ``response_body`` - the plain response body
|
||||||
|
|
||||||
* ``response_headers`` - the response headers (in raw HTTP format)
|
* ``response_headers`` - the request headers (in raw HTTP format)
|
||||||
|
|
||||||
* ``meta`` - some metadata of this cache resource in Python ``repr()``
|
* ``meta`` - some metadata of this cache resource in Python ``repr()``
|
||||||
format (grep-friendly format)
|
format (grep-friendly format)
|
||||||
|
|
@ -547,7 +547,7 @@ defines the methods described below.
|
||||||
.. method:: open_spider(spider)
|
.. method:: open_spider(spider)
|
||||||
|
|
||||||
This method gets called after a spider has been opened for crawling. It handles
|
This method gets called after a spider has been opened for crawling. It handles
|
||||||
the :signal:`spider_opened` signal.
|
the :signal:`open_spider <spider_opened>` signal.
|
||||||
|
|
||||||
:param spider: the spider which has been opened
|
:param spider: the spider which has been opened
|
||||||
:type spider: :class:`~scrapy.Spider` object
|
:type spider: :class:`~scrapy.Spider` object
|
||||||
|
|
@ -555,7 +555,7 @@ defines the methods described below.
|
||||||
.. method:: close_spider(spider)
|
.. method:: close_spider(spider)
|
||||||
|
|
||||||
This method gets called after a spider has been closed. It handles
|
This method gets called after a spider has been closed. It handles
|
||||||
the :signal:`spider_closed` signal.
|
the :signal:`close_spider <spider_closed>` signal.
|
||||||
|
|
||||||
:param spider: the spider which has been closed
|
:param spider: the spider which has been closed
|
||||||
:type spider: :class:`~scrapy.Spider` object
|
:type spider: :class:`~scrapy.Spider` object
|
||||||
|
|
@ -564,10 +564,6 @@ defines the methods described below.
|
||||||
|
|
||||||
Return response if present in cache, or ``None`` otherwise.
|
Return response if present in cache, or ``None`` otherwise.
|
||||||
|
|
||||||
If this method raises an exception, e.g. because the cache entry is
|
|
||||||
corrupted, the middleware logs a warning and handles the request as a
|
|
||||||
cache miss.
|
|
||||||
|
|
||||||
:param spider: the spider which generated the request
|
:param spider: the spider which generated the request
|
||||||
:type spider: :class:`~scrapy.Spider` object
|
:type spider: :class:`~scrapy.Spider` object
|
||||||
|
|
||||||
|
|
@ -595,8 +591,8 @@ In order to use your storage backend, set:
|
||||||
HTTPCache middleware settings
|
HTTPCache middleware settings
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` can be
|
The :class:`HttpCacheMiddleware` can be configured through the following
|
||||||
configured through the following settings:
|
settings:
|
||||||
|
|
||||||
.. setting:: HTTPCACHE_ENABLED
|
.. setting:: HTTPCACHE_ENABLED
|
||||||
|
|
||||||
|
|
@ -731,8 +727,6 @@ We assume that the spider will not issue Cache-Control directives
|
||||||
in requests unless it actually needs them, so directives in requests are
|
in requests unless it actually needs them, so directives in requests are
|
||||||
not filtered.
|
not filtered.
|
||||||
|
|
||||||
.. _http-compression:
|
|
||||||
|
|
||||||
HttpCompressionMiddleware
|
HttpCompressionMiddleware
|
||||||
-------------------------
|
-------------------------
|
||||||
|
|
||||||
|
|
@ -744,12 +738,14 @@ HttpCompressionMiddleware
|
||||||
This middleware allows compressed (gzip, deflate) traffic to be
|
This middleware allows compressed (gzip, deflate) traffic to be
|
||||||
sent/received from web sites.
|
sent/received from web sites.
|
||||||
|
|
||||||
This middleware also supports decoding `brotli-compressed`_ responses with
|
This middleware also supports decoding `brotli-compressed`_ as well as
|
||||||
the :ref:`brotli <extras>` extra, and `zstd-compressed`_
|
`zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is
|
||||||
responses with the :ref:`zstd <extras>` extra.
|
installed, respectively.
|
||||||
|
|
||||||
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
|
.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
|
||||||
|
.. _brotli: https://pypi.org/project/Brotli/
|
||||||
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
|
.. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
|
||||||
|
.. _zstandard: https://pypi.org/project/zstandard/
|
||||||
|
|
||||||
|
|
||||||
HttpCompressionMiddleware Settings
|
HttpCompressionMiddleware Settings
|
||||||
|
|
@ -818,6 +814,7 @@ HttpProxyMiddleware settings
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
.. setting:: HTTPPROXY_ENABLED
|
.. setting:: HTTPPROXY_ENABLED
|
||||||
|
.. setting:: HTTPPROXY_AUTH_ENCODING
|
||||||
|
|
||||||
HTTPPROXY_ENABLED
|
HTTPPROXY_ENABLED
|
||||||
^^^^^^^^^^^^^^^^^
|
^^^^^^^^^^^^^^^^^
|
||||||
|
|
@ -826,8 +823,6 @@ Default: ``True``
|
||||||
|
|
||||||
Whether or not to enable the :class:`HttpProxyMiddleware`.
|
Whether or not to enable the :class:`HttpProxyMiddleware`.
|
||||||
|
|
||||||
.. setting:: HTTPPROXY_AUTH_ENCODING
|
|
||||||
|
|
||||||
HTTPPROXY_AUTH_ENCODING
|
HTTPPROXY_AUTH_ENCODING
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^
|
^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
|
|
@ -872,9 +867,9 @@ OffsiteMiddleware
|
||||||
.. reqmeta:: allow_offsite
|
.. reqmeta:: allow_offsite
|
||||||
|
|
||||||
If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to
|
If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to
|
||||||
``True`` or :attr:`Request.meta <scrapy.Request.meta>` has ``allow_offsite``
|
``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then
|
||||||
set to ``True``, then the OffsiteMiddleware will allow the request even if
|
the OffsiteMiddleware will allow the request even if its domain is not listed
|
||||||
its domain is not listed in allowed domains.
|
in allowed domains.
|
||||||
|
|
||||||
RedirectMiddleware
|
RedirectMiddleware
|
||||||
------------------
|
------------------
|
||||||
|
|
@ -989,7 +984,7 @@ Whether the Meta Refresh middleware will be enabled.
|
||||||
METAREFRESH_IGNORE_TAGS
|
METAREFRESH_IGNORE_TAGS
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^
|
^^^^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
Default: ``["noscript"]``
|
Default: ``[]``
|
||||||
|
|
||||||
Meta tags within these tags are ignored.
|
Meta tags within these tags are ignored.
|
||||||
|
|
||||||
|
|
@ -1019,6 +1014,17 @@ RetryMiddleware
|
||||||
A middleware to retry failed requests that are potentially caused by
|
A middleware to retry failed requests that are potentially caused by
|
||||||
temporary problems such as a connection timeout or HTTP 500 error.
|
temporary problems such as a connection timeout or HTTP 500 error.
|
||||||
|
|
||||||
|
Failed pages are collected on the scraping process and rescheduled at the
|
||||||
|
end, once the spider has finished crawling all regular (non failed) pages.
|
||||||
|
|
||||||
|
The :class:`RetryMiddleware` can be configured through the following
|
||||||
|
settings (see the settings documentation for more info):
|
||||||
|
|
||||||
|
* :setting:`RETRY_ENABLED`
|
||||||
|
* :setting:`RETRY_TIMES`
|
||||||
|
* :setting:`RETRY_HTTP_CODES`
|
||||||
|
* :setting:`RETRY_EXCEPTIONS`
|
||||||
|
|
||||||
.. reqmeta:: dont_retry
|
.. reqmeta:: dont_retry
|
||||||
|
|
||||||
If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_retry`` key
|
If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_retry`` key
|
||||||
|
|
@ -1085,7 +1091,7 @@ Default::
|
||||||
'twisted.internet.error.ConnectionDone',
|
'twisted.internet.error.ConnectionDone',
|
||||||
'twisted.internet.error.ConnectError',
|
'twisted.internet.error.ConnectError',
|
||||||
'twisted.internet.error.ConnectionLost',
|
'twisted.internet.error.ConnectionLost',
|
||||||
OSError,
|
IOError,
|
||||||
'scrapy.core.downloader.handlers.http11.TunnelError',
|
'scrapy.core.downloader.handlers.http11.TunnelError',
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -1243,7 +1249,8 @@ Based on `Robotexclusionrulesparser <https://pypi.org/project/robotexclusionrule
|
||||||
|
|
||||||
In order to use this parser:
|
In order to use this parser:
|
||||||
|
|
||||||
* Install the :ref:`robotparser <extras>` extra.
|
* Install ``Robotexclusionrulesparser`` by running
|
||||||
|
``pip install robotexclusionrulesparser``
|
||||||
|
|
||||||
* Set :setting:`ROBOTSTXT_PARSER` setting to
|
* Set :setting:`ROBOTSTXT_PARSER` setting to
|
||||||
``scrapy.robotstxt.RerpRobotParser``
|
``scrapy.robotstxt.RerpRobotParser``
|
||||||
|
|
|
||||||
|
|
@ -133,7 +133,7 @@ data from it depends on the type of response:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
selector = Selector(text=data["html"])
|
selector = Selector(data["html"])
|
||||||
|
|
||||||
- If the response is JavaScript, or HTML with a ``<script/>`` element
|
- If the response is JavaScript, or HTML with a ``<script/>`` element
|
||||||
containing the desired data, see :ref:`topics-parsing-javascript`.
|
containing the desired data, see :ref:`topics-parsing-javascript`.
|
||||||
|
|
|
||||||
|
|
@ -1,25 +1,115 @@
|
||||||
.. _topics-exceptions:
|
.. _topics-exceptions:
|
||||||
.. _topics-exceptions-ref:
|
|
||||||
|
|
||||||
==========
|
==========
|
||||||
Exceptions
|
Exceptions
|
||||||
==========
|
==========
|
||||||
|
|
||||||
Here's a list of all exceptions included in Scrapy and their usage, except for
|
|
||||||
the :ref:`download handler exceptions <download-handlers-exceptions>`.
|
|
||||||
|
|
||||||
.. module:: scrapy.exceptions
|
.. module:: scrapy.exceptions
|
||||||
|
:synopsis: Scrapy exceptions
|
||||||
|
|
||||||
.. autoexception:: CloseSpider
|
.. _topics-exceptions-ref:
|
||||||
|
|
||||||
.. autoexception:: DontCloseSpider
|
Built-in Exceptions reference
|
||||||
|
=============================
|
||||||
|
|
||||||
.. autoexception:: DropItem
|
Here's a list of all exceptions included in Scrapy and their usage.
|
||||||
|
|
||||||
.. autoexception:: IgnoreRequest
|
|
||||||
|
|
||||||
.. autoexception:: NotConfigured
|
CloseSpider
|
||||||
|
-----------
|
||||||
|
|
||||||
.. autoexception:: NotSupported
|
.. exception:: CloseSpider(reason='cancelled')
|
||||||
|
|
||||||
.. autoexception:: StopDownload
|
This exception can be raised from a spider callback to request the spider to be
|
||||||
|
closed/stopped. Supported arguments:
|
||||||
|
|
||||||
|
:param reason: the reason for closing
|
||||||
|
:type reason: str
|
||||||
|
|
||||||
|
For example:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
def parse_page(self, response):
|
||||||
|
if "Bandwidth exceeded" in response.body:
|
||||||
|
raise CloseSpider("bandwidth_exceeded")
|
||||||
|
|
||||||
|
DontCloseSpider
|
||||||
|
---------------
|
||||||
|
|
||||||
|
.. exception:: DontCloseSpider
|
||||||
|
|
||||||
|
This exception can be raised in a :signal:`spider_idle` signal handler to
|
||||||
|
prevent the spider from being closed.
|
||||||
|
|
||||||
|
DropItem
|
||||||
|
--------
|
||||||
|
|
||||||
|
.. exception:: DropItem
|
||||||
|
|
||||||
|
The exception that must be raised by item pipeline stages to stop processing an
|
||||||
|
Item. For more information see :ref:`topics-item-pipeline`.
|
||||||
|
|
||||||
|
IgnoreRequest
|
||||||
|
-------------
|
||||||
|
|
||||||
|
.. exception:: IgnoreRequest
|
||||||
|
|
||||||
|
This exception can be raised by the Scheduler or any downloader middleware to
|
||||||
|
indicate that the request should be ignored.
|
||||||
|
|
||||||
|
NotConfigured
|
||||||
|
-------------
|
||||||
|
|
||||||
|
.. exception:: NotConfigured
|
||||||
|
|
||||||
|
This exception can be raised by some components to indicate that they will
|
||||||
|
remain disabled. Those components include:
|
||||||
|
|
||||||
|
- Extensions
|
||||||
|
- Item pipelines
|
||||||
|
- Downloader middlewares
|
||||||
|
- Spider middlewares
|
||||||
|
|
||||||
|
The exception must be raised in the component's ``__init__`` method.
|
||||||
|
|
||||||
|
NotSupported
|
||||||
|
------------
|
||||||
|
|
||||||
|
.. exception:: NotSupported
|
||||||
|
|
||||||
|
This exception is raised to indicate an unsupported feature.
|
||||||
|
|
||||||
|
StopDownload
|
||||||
|
-------------
|
||||||
|
|
||||||
|
.. exception:: StopDownload(fail=True)
|
||||||
|
|
||||||
|
Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
|
||||||
|
signal handler to indicate that no further bytes should be downloaded for a response.
|
||||||
|
|
||||||
|
The ``fail`` boolean parameter controls which method will handle the resulting
|
||||||
|
response:
|
||||||
|
|
||||||
|
* If ``fail=True`` (default), the request errback is called. The response object is
|
||||||
|
available as the ``response`` attribute of the ``StopDownload`` exception,
|
||||||
|
which is in turn stored as the ``value`` attribute of the received
|
||||||
|
:class:`~twisted.python.failure.Failure` object. This means that in an errback
|
||||||
|
defined as ``def errback(self, failure)``, the response can be accessed though
|
||||||
|
``failure.value.response``.
|
||||||
|
|
||||||
|
* If ``fail=False``, the request callback is called instead.
|
||||||
|
|
||||||
|
In both cases, the response could have its body truncated: the body contains
|
||||||
|
all bytes received up until the exception is raised, including the bytes
|
||||||
|
received in the signal handler that raises the exception. Also, the response
|
||||||
|
object is marked with ``"download_stopped"`` in its :attr:`~scrapy.http.Response.flags`
|
||||||
|
attribute.
|
||||||
|
|
||||||
|
.. note:: ``fail`` is a keyword-only parameter, i.e. raising
|
||||||
|
``StopDownload(False)`` or ``StopDownload(True)`` will raise
|
||||||
|
a :class:`TypeError`.
|
||||||
|
|
||||||
|
See the documentation for the :class:`~scrapy.signals.bytes_received` and
|
||||||
|
:class:`~scrapy.signals.headers_received` signals
|
||||||
|
and the :ref:`topics-stop-response-download` topic for additional information and examples.
|
||||||
|
|
|
||||||
|
|
@ -153,7 +153,7 @@ output examples, which assume you're exporting these two items:
|
||||||
BaseItemExporter
|
BaseItemExporter
|
||||||
----------------
|
----------------
|
||||||
|
|
||||||
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding=None, indent=None, dont_fail=False)
|
.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0, dont_fail=False)
|
||||||
|
|
||||||
This is the (abstract) base class for all Item Exporters. It provides
|
This is the (abstract) base class for all Item Exporters. It provides
|
||||||
support for common features used by all (concrete) Item Exporters, such as
|
support for common features used by all (concrete) Item Exporters, such as
|
||||||
|
|
@ -211,27 +211,13 @@ BaseItemExporter
|
||||||
|
|
||||||
- ``None`` (all fields [2]_, default)
|
- ``None`` (all fields [2]_, default)
|
||||||
|
|
||||||
Fields are exported in declaration order, i.e. the order in which
|
- A list of fields::
|
||||||
they are defined in the :ref:`item class <item-types>`. For
|
|
||||||
:class:`dict` items, which have no declared fields, the key order of
|
|
||||||
each item is used instead.
|
|
||||||
|
|
||||||
.. versionchanged:: VERSION
|
['field1', 'field2']
|
||||||
Fields of non-\ :class:`dict` items used to be exported in the
|
|
||||||
order in which they had been populated, except in
|
|
||||||
:class:`CsvItemExporter`, which has always used declaration order.
|
|
||||||
|
|
||||||
- A list of fields:
|
- A dict where keys are fields and values are output names::
|
||||||
|
|
||||||
.. code-block:: python
|
{'field1': 'Field 1', 'field2': 'Field 2'}
|
||||||
|
|
||||||
["field1", "field2"]
|
|
||||||
|
|
||||||
- A dict where keys are fields and values are output names:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
{"field1": "Field 1", "field2": "Field 2"}
|
|
||||||
|
|
||||||
.. [1] Not all exporters respect the specified field order.
|
.. [1] Not all exporters respect the specified field order.
|
||||||
.. [2] When using :ref:`item objects <item-types>` that do not expose
|
.. [2] When using :ref:`item objects <item-types>` that do not expose
|
||||||
|
|
@ -253,7 +239,7 @@ BaseItemExporter
|
||||||
|
|
||||||
.. attribute:: indent
|
.. attribute:: indent
|
||||||
|
|
||||||
Amount of spaces used to indent the output on each level. Defaults to ``None``.
|
Amount of spaces used to indent the output on each level. Defaults to ``0``.
|
||||||
|
|
||||||
* ``indent=None`` selects the most compact representation,
|
* ``indent=None`` selects the most compact representation,
|
||||||
all items in the same line with no indentation
|
all items in the same line with no indentation
|
||||||
|
|
@ -287,9 +273,7 @@ XmlItemExporter
|
||||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||||
:class:`BaseItemExporter` ``__init__`` method.
|
:class:`BaseItemExporter` ``__init__`` method.
|
||||||
|
|
||||||
A typical output of this exporter would be:
|
A typical output of this exporter would be::
|
||||||
|
|
||||||
.. code-block:: xml
|
|
||||||
|
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<items>
|
<items>
|
||||||
|
|
@ -307,17 +291,11 @@ XmlItemExporter
|
||||||
exported by serializing each value inside a ``<value>`` element. This is for
|
exported by serializing each value inside a ``<value>`` element. This is for
|
||||||
convenience, as multi-valued fields are very common.
|
convenience, as multi-valued fields are very common.
|
||||||
|
|
||||||
For example, the item:
|
For example, the item::
|
||||||
|
|
||||||
.. skip: next
|
Item(name=['John', 'Doe'], age='23')
|
||||||
|
|
||||||
.. code-block:: python
|
Would be serialized as::
|
||||||
|
|
||||||
Item(name=["John", "Doe"], age="23")
|
|
||||||
|
|
||||||
Would be serialized as:
|
|
||||||
|
|
||||||
.. code-block:: xml
|
|
||||||
|
|
||||||
<?xml version="1.0" encoding="utf-8"?>
|
<?xml version="1.0" encoding="utf-8"?>
|
||||||
<items>
|
<items>
|
||||||
|
|
@ -350,7 +328,7 @@ CsvItemExporter
|
||||||
|
|
||||||
:param join_multivalued: The char (or chars) that will be used for joining
|
:param join_multivalued: The char (or chars) that will be used for joining
|
||||||
multi-valued fields, if found.
|
multi-valued fields, if found.
|
||||||
:type join_multivalued: str
|
:type include_headers_line: str
|
||||||
|
|
||||||
:param errors: The optional string that specifies how encoding and decoding
|
:param errors: The optional string that specifies how encoding and decoding
|
||||||
errors are to be handled. For more information see
|
errors are to be handled. For more information see
|
||||||
|
|
@ -364,14 +342,14 @@ CsvItemExporter
|
||||||
|
|
||||||
A typical output of this exporter would be::
|
A typical output of this exporter would be::
|
||||||
|
|
||||||
name,price
|
product,price
|
||||||
Color TV,1200
|
Color TV,1200
|
||||||
DVD player,200
|
DVD player,200
|
||||||
|
|
||||||
PickleItemExporter
|
PickleItemExporter
|
||||||
------------------
|
------------------
|
||||||
|
|
||||||
.. class:: PickleItemExporter(file, protocol=4, **kwargs)
|
.. class:: PickleItemExporter(file, protocol=0, **kwargs)
|
||||||
|
|
||||||
Exports items in pickle format to the given file-like object.
|
Exports items in pickle format to the given file-like object.
|
||||||
|
|
||||||
|
|
@ -401,12 +379,10 @@ PprintItemExporter
|
||||||
The additional keyword arguments of this ``__init__`` method are passed to the
|
The additional keyword arguments of this ``__init__`` method are passed to the
|
||||||
:class:`BaseItemExporter` ``__init__`` method.
|
:class:`BaseItemExporter` ``__init__`` method.
|
||||||
|
|
||||||
A typical output of this exporter would be:
|
A typical output of this exporter would be::
|
||||||
|
|
||||||
.. code-block:: python
|
{'name': 'Color TV', 'price': '1200'}
|
||||||
|
{'name': 'DVD player', 'price': '200'}
|
||||||
{"name": "Color TV", "price": "1200"}
|
|
||||||
{"name": "DVD player", "price": "200"}
|
|
||||||
|
|
||||||
Longer lines (when present) are pretty-formatted.
|
Longer lines (when present) are pretty-formatted.
|
||||||
|
|
||||||
|
|
@ -424,9 +400,7 @@ JsonItemExporter
|
||||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||||
|
|
||||||
A typical output of this exporter would be:
|
A typical output of this exporter would be::
|
||||||
|
|
||||||
.. code-block:: json
|
|
||||||
|
|
||||||
[{"name": "Color TV", "price": "1200"},
|
[{"name": "Color TV", "price": "1200"},
|
||||||
{"name": "DVD player", "price": "200"}]
|
{"name": "DVD player", "price": "200"}]
|
||||||
|
|
@ -455,9 +429,7 @@ JsonLinesItemExporter
|
||||||
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
:param file: the file-like object to use for exporting the data. Its ``write`` method should
|
||||||
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
|
||||||
|
|
||||||
A typical output of this exporter would be:
|
A typical output of this exporter would be::
|
||||||
|
|
||||||
.. code-block:: json
|
|
||||||
|
|
||||||
{"name": "Color TV", "price": "1200"}
|
{"name": "Color TV", "price": "1200"}
|
||||||
{"name": "DVD player", "price": "200"}
|
{"name": "DVD player", "price": "200"}
|
||||||
|
|
|
||||||
|
|
@ -136,10 +136,18 @@ Core Stats extension
|
||||||
Enable the collection of core statistics, provided the stats collection is
|
Enable the collection of core statistics, provided the stats collection is
|
||||||
enabled (see :ref:`topics-stats`).
|
enabled (see :ref:`topics-stats`).
|
||||||
|
|
||||||
The following stats are collected: :stat:`elapsed_time_seconds`,
|
The following stats are collected:
|
||||||
:stat:`finish_reason`, :stat:`finish_time`, :stat:`item_dropped_count`,
|
|
||||||
:stat:`item_dropped_reasons_count/{exception}`, :stat:`item_scraped_count`,
|
* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`).
|
||||||
:stat:`response_received_count`, :stat:`start_time`.
|
* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`).
|
||||||
|
* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`).
|
||||||
|
* ``finish_reason``: the closing reason string (e.g. ``"finished"``,
|
||||||
|
``"closespider_timeout"``).
|
||||||
|
* ``item_scraped_count``: total number of items that passed all pipelines.
|
||||||
|
* ``item_dropped_count``: total number of items dropped by a pipeline.
|
||||||
|
* ``item_dropped_reasons_count/<ExceptionName>``: per-exception drop count
|
||||||
|
(e.g. ``item_dropped_reasons_count/DropItem``).
|
||||||
|
* ``response_received_count``: total number of HTTP responses received.
|
||||||
|
|
||||||
Log Count extension
|
Log Count extension
|
||||||
~~~~~~~~~~~~~~~~~~~
|
~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
@ -182,7 +190,7 @@ Monitors the memory used by the Scrapy process that runs the spider and:
|
||||||
|
|
||||||
1. sends a :signal:`memusage_warning_reached` signal when it exceeds
|
1. sends a :signal:`memusage_warning_reached` signal when it exceeds
|
||||||
:setting:`MEMUSAGE_WARNING_MB`
|
:setting:`MEMUSAGE_WARNING_MB`
|
||||||
2. closes the spider with the ``"memusage_exceeded"`` reason when it exceeds
|
2. closes the spider with the `"memusage_exceeded"` reason when it exceeds
|
||||||
:setting:`MEMUSAGE_LIMIT_MB`
|
:setting:`MEMUSAGE_LIMIT_MB`
|
||||||
|
|
||||||
This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and
|
This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and
|
||||||
|
|
@ -206,8 +214,7 @@ An extension for debugging memory usage. It collects information about:
|
||||||
* objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs`
|
* objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs`
|
||||||
|
|
||||||
To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The
|
To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The
|
||||||
info will be stored in the :stat:`memdebug/gc_garbage_count` and
|
info will be stored in the stats.
|
||||||
:stat:`memdebug/live_refs/{cls}` stats.
|
|
||||||
|
|
||||||
.. _topics-extensions-ref-spiderstate:
|
.. _topics-extensions-ref-spiderstate:
|
||||||
|
|
||||||
|
|
@ -334,6 +341,9 @@ closing the spider. If the spider generates more than that number of errors,
|
||||||
it will be closed with the reason ``closespider_errorcount``. If zero (or non
|
it will be closed with the reason ``closespider_errorcount``. If zero (or non
|
||||||
set), spiders won't be closed by number of errors.
|
set), spiders won't be closed by number of errors.
|
||||||
|
|
||||||
|
.. module:: scrapy.extensions.debug
|
||||||
|
:synopsis: Extensions for debugging Scrapy
|
||||||
|
|
||||||
.. module:: scrapy.extensions.periodic_log
|
.. module:: scrapy.extensions.periodic_log
|
||||||
:synopsis: Periodic stats logging
|
:synopsis: Periodic stats logging
|
||||||
|
|
||||||
|
|
@ -408,7 +418,7 @@ Example extension configuration:
|
||||||
custom_settings = {
|
custom_settings = {
|
||||||
"LOG_LEVEL": "INFO",
|
"LOG_LEVEL": "INFO",
|
||||||
"PERIODIC_LOG_STATS": {
|
"PERIODIC_LOG_STATS": {
|
||||||
"include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count"],
|
"include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count/"],
|
||||||
},
|
},
|
||||||
"PERIODIC_LOG_DELTA": {"include": ["downloader/"]},
|
"PERIODIC_LOG_DELTA": {"include": ["downloader/"]},
|
||||||
"PERIODIC_LOG_TIMING_ENABLED": True,
|
"PERIODIC_LOG_TIMING_ENABLED": True,
|
||||||
|
|
@ -453,9 +463,6 @@ Default: ``False``
|
||||||
Debugging extensions
|
Debugging extensions
|
||||||
--------------------
|
--------------------
|
||||||
|
|
||||||
.. module:: scrapy.extensions.debug
|
|
||||||
:synopsis: Extensions for debugging Scrapy
|
|
||||||
|
|
||||||
Stack trace dump extension
|
Stack trace dump extension
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -92,6 +92,7 @@ Marshal
|
||||||
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
|
- Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
|
||||||
- Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
|
- Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
|
||||||
|
|
||||||
|
|
||||||
.. _topics-feed-storage:
|
.. _topics-feed-storage:
|
||||||
|
|
||||||
Storages
|
Storages
|
||||||
|
|
@ -105,13 +106,14 @@ The storages backends supported out of the box are:
|
||||||
|
|
||||||
- :ref:`topics-feed-storage-fs`
|
- :ref:`topics-feed-storage-fs`
|
||||||
- :ref:`topics-feed-storage-ftp`
|
- :ref:`topics-feed-storage-ftp`
|
||||||
- :ref:`topics-feed-storage-s3` (requires the :ref:`s3 <extras>` extra)
|
- :ref:`topics-feed-storage-s3` (requires boto3_)
|
||||||
- :ref:`topics-feed-storage-gcs` (requires the :ref:`gcs <extras>` extra)
|
- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
|
||||||
- :ref:`topics-feed-storage-stdout`
|
- :ref:`topics-feed-storage-stdout`
|
||||||
|
|
||||||
Some storage backends may be unavailable if the required :ref:`extras <extras>`
|
Some storage backends may be unavailable if the required external libraries are
|
||||||
are not installed. For example, the S3 backend requires the :ref:`s3 <extras>`
|
not available. For example, the S3 backend is only available if the boto3_
|
||||||
extra.
|
library is installed.
|
||||||
|
|
||||||
|
|
||||||
.. _topics-feed-uri-params:
|
.. _topics-feed-uri-params:
|
||||||
|
|
||||||
|
|
@ -141,11 +143,6 @@ Here are some examples to illustrate:
|
||||||
.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
|
.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
|
||||||
they can also be used as storage URI parameters.
|
they can also be used as storage URI parameters.
|
||||||
|
|
||||||
.. note:: Only ``%(...)s`` parameters are replaced. Any other percent
|
|
||||||
character is kept as-is, so percent-encoded URIs (e.g. ``%20`` for a
|
|
||||||
space or percent-encoded FTP credentials) and :class:`pathlib.Path`
|
|
||||||
keys containing ``%(...)s`` parameters both work as expected.
|
|
||||||
|
|
||||||
|
|
||||||
.. _topics-feed-storage-backends:
|
.. _topics-feed-storage-backends:
|
||||||
|
|
||||||
|
|
@ -164,7 +161,7 @@ The feeds are stored in the local filesystem.
|
||||||
- Required external libraries: none
|
- Required external libraries: none
|
||||||
|
|
||||||
Note that for the local filesystem storage (only) you can omit the scheme if
|
Note that for the local filesystem storage (only) you can omit the scheme if
|
||||||
you specify a path (e.g. ``/tmp/export.csv``).
|
you specify an absolute path like ``/tmp/export.csv`` (Unix systems only).
|
||||||
Alternatively you can also use a :class:`pathlib.Path` object.
|
Alternatively you can also use a :class:`pathlib.Path` object.
|
||||||
|
|
||||||
.. _topics-feed-storage-ftp:
|
.. _topics-feed-storage-ftp:
|
||||||
|
|
@ -207,7 +204,7 @@ The feeds are stored on `Amazon S3`_.
|
||||||
|
|
||||||
- ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
|
- ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
|
||||||
|
|
||||||
- Required extras: :ref:`s3 <extras>`
|
- Required external libraries: `boto3`_ >= 1.20.0
|
||||||
|
|
||||||
The AWS credentials can be passed as user/password in the URI, or they can be
|
The AWS credentials can be passed as user/password in the URI, or they can be
|
||||||
passed through the following settings:
|
passed through the following settings:
|
||||||
|
|
@ -218,13 +215,12 @@ passed through the following settings:
|
||||||
|
|
||||||
.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
|
.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
|
||||||
|
|
||||||
You can also define a custom ACL, custom endpoint, region name and connection
|
You can also define a custom ACL, custom endpoint, and region name for exported
|
||||||
pool size for exported feeds using these settings:
|
feeds using these settings:
|
||||||
|
|
||||||
- :setting:`FEED_STORAGE_S3_ACL`
|
- :setting:`FEED_STORAGE_S3_ACL`
|
||||||
- :setting:`AWS_ENDPOINT_URL`
|
- :setting:`AWS_ENDPOINT_URL`
|
||||||
- :setting:`AWS_REGION_NAME`
|
- :setting:`AWS_REGION_NAME`
|
||||||
- :setting:`AWS_MAX_POOL_CONNECTIONS`
|
|
||||||
|
|
||||||
The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
|
The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
|
||||||
storage backend is: ``True``.
|
storage backend is: ``True``.
|
||||||
|
|
@ -248,7 +244,7 @@ The feeds are stored on `Google Cloud Storage`_.
|
||||||
|
|
||||||
- ``gs://mybucket/path/to/export.csv``
|
- ``gs://mybucket/path/to/export.csv``
|
||||||
|
|
||||||
- Required extras: :ref:`gcs <extras>`
|
- Required external libraries: `google-cloud-storage`_.
|
||||||
|
|
||||||
For more information about authentication, please refer to `Google Cloud documentation <https://docs.cloud.google.com/docs/authentication>`_.
|
For more information about authentication, please refer to `Google Cloud documentation <https://docs.cloud.google.com/docs/authentication>`_.
|
||||||
|
|
||||||
|
|
@ -265,6 +261,7 @@ storage backend is: ``True``.
|
||||||
|
|
||||||
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
|
This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
|
||||||
|
|
||||||
|
.. _google-cloud-storage: https://docs.cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
|
||||||
|
|
||||||
|
|
||||||
.. _topics-feed-storage-stdout:
|
.. _topics-feed-storage-stdout:
|
||||||
|
|
@ -432,37 +429,33 @@ This setting is required for enabling the feed export feature.
|
||||||
|
|
||||||
See :ref:`topics-feed-storage-backends` for supported URI schemes.
|
See :ref:`topics-feed-storage-backends` for supported URI schemes.
|
||||||
|
|
||||||
For instance:
|
For instance::
|
||||||
|
|
||||||
.. skip: next
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
{
|
{
|
||||||
"items.json": {
|
'items.json': {
|
||||||
"format": "json",
|
'format': 'json',
|
||||||
"encoding": "utf8",
|
'encoding': 'utf8',
|
||||||
"store_empty": False,
|
'store_empty': False,
|
||||||
"item_classes": [MyItemClass1, "myproject.items.MyItemClass2"],
|
'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'],
|
||||||
"fields": None,
|
'fields': None,
|
||||||
"indent": 4,
|
'indent': 4,
|
||||||
"item_export_kwargs": {
|
'item_export_kwargs': {
|
||||||
"export_empty_fields": True,
|
'export_empty_fields': True,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"/home/user/documents/items.xml": {
|
'/home/user/documents/items.xml': {
|
||||||
"format": "xml",
|
'format': 'xml',
|
||||||
"fields": ["name", "price"],
|
'fields': ['name', 'price'],
|
||||||
"item_filter": MyCustomFilter1,
|
'item_filter': MyCustomFilter1,
|
||||||
"encoding": "latin1",
|
'encoding': 'latin1',
|
||||||
"indent": 8,
|
'indent': 8,
|
||||||
},
|
},
|
||||||
pathlib.Path("items.csv.gz"): {
|
pathlib.Path('items.csv.gz'): {
|
||||||
"format": "csv",
|
'format': 'csv',
|
||||||
"fields": ["price", "name"],
|
'fields': ['price', 'name'],
|
||||||
"item_filter": "myproject.filters.MyCustomFilter2",
|
'item_filter': 'myproject.filters.MyCustomFilter2',
|
||||||
"postprocessing": [MyPlugin1, "scrapy.extensions.postprocessing.GzipPlugin"],
|
'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
|
||||||
"gzip_compresslevel": 5,
|
'gzip_compresslevel': 5,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -622,7 +615,6 @@ Default:
|
||||||
"file": "scrapy.extensions.feedexport.FileFeedStorage",
|
"file": "scrapy.extensions.feedexport.FileFeedStorage",
|
||||||
"stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
|
"stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
|
||||||
"s3": "scrapy.extensions.feedexport.S3FeedStorage",
|
"s3": "scrapy.extensions.feedexport.S3FeedStorage",
|
||||||
"gs": "scrapy.extensions.feedexport.GCSFeedStorage",
|
|
||||||
"ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
|
"ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -764,8 +756,8 @@ The function signature should be as follows:
|
||||||
:param spider: source spider of the feed items
|
:param spider: source spider of the feed items
|
||||||
:type spider: scrapy.Spider
|
:type spider: scrapy.Spider
|
||||||
|
|
||||||
.. caution:: The function must return a new dictionary instead of modifying
|
.. caution:: The function should return a new dictionary, modifying
|
||||||
the received ``params`` in-place.
|
the received ``params`` in-place is deprecated.
|
||||||
|
|
||||||
For example, to include the :attr:`name <scrapy.Spider.name>` of the
|
For example, to include the :attr:`name <scrapy.Spider.name>` of the
|
||||||
source spider in the feed URI:
|
source spider in the feed URI:
|
||||||
|
|
@ -792,5 +784,6 @@ source spider in the feed URI:
|
||||||
|
|
||||||
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
|
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
|
||||||
.. _Amazon S3: https://aws.amazon.com/s3/
|
.. _Amazon S3: https://aws.amazon.com/s3/
|
||||||
|
.. _boto3: https://github.com/boto/boto3
|
||||||
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
|
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
|
||||||
.. _Google Cloud Storage: https://cloud.google.com/storage/
|
.. _Google Cloud Storage: https://cloud.google.com/storage/
|
||||||
|
|
|
||||||
|
|
@ -121,7 +121,7 @@ Write items to MongoDB
|
||||||
|
|
||||||
In this example we'll write items to MongoDB_ using pymongo_.
|
In this example we'll write items to MongoDB_ using pymongo_.
|
||||||
MongoDB address and database name are specified in Scrapy settings;
|
MongoDB address and database name are specified in Scrapy settings;
|
||||||
MongoDB collection is specified in a class attribute.
|
MongoDB collection is named after item class.
|
||||||
|
|
||||||
The main point of this example is to show how to :ref:`get the crawler
|
The main point of this example is to show how to :ref:`get the crawler
|
||||||
<from-crawler>` and how to clean up the resources properly.
|
<from-crawler>` and how to clean up the resources properly.
|
||||||
|
|
|
||||||
|
|
@ -23,8 +23,7 @@ Item Types
|
||||||
|
|
||||||
Scrapy supports the following types of items, via the `itemadapter`_ library:
|
Scrapy supports the following types of items, via the `itemadapter`_ library:
|
||||||
:ref:`dictionaries <dict-items>`, :ref:`Item objects <item-objects>`,
|
:ref:`dictionaries <dict-items>`, :ref:`Item objects <item-objects>`,
|
||||||
:ref:`dataclass objects <dataclass-items>`, :ref:`attrs objects <attrs-items>`
|
:ref:`dataclass objects <dataclass-items>`, and :ref:`attrs objects <attrs-items>`.
|
||||||
and :ref:`Pydantic models <pydantic-items>`.
|
|
||||||
|
|
||||||
.. _itemadapter: https://github.com/scrapy/itemadapter
|
.. _itemadapter: https://github.com/scrapy/itemadapter
|
||||||
|
|
||||||
|
|
@ -62,8 +61,8 @@ its ``__init__`` method.
|
||||||
:class:`Item` also allows the defining of field metadata, which can be used to
|
:class:`Item` also allows the defining of field metadata, which can be used to
|
||||||
:ref:`customize serialization <topics-exporters-field-serialization>`.
|
:ref:`customize serialization <topics-exporters-field-serialization>`.
|
||||||
|
|
||||||
:mod:`scrapy.utils.trackref` tracks :class:`Item` objects to help find memory
|
:mod:`trackref` tracks :class:`Item` objects to help find memory leaks
|
||||||
leaks (see :ref:`topics-leaks-trackrefs`).
|
(see :ref:`topics-leaks-trackrefs`).
|
||||||
|
|
||||||
Example:
|
Example:
|
||||||
|
|
||||||
|
|
@ -263,7 +262,7 @@ Creating items
|
||||||
|
|
||||||
>>> product = Product(name="Desktop PC", price=1000)
|
>>> product = Product(name="Desktop PC", price=1000)
|
||||||
>>> print(product)
|
>>> print(product)
|
||||||
{'name': 'Desktop PC', 'price': 1000}
|
Product(name='Desktop PC', price=1000)
|
||||||
|
|
||||||
|
|
||||||
Getting field values
|
Getting field values
|
||||||
|
|
@ -377,12 +376,10 @@ Creating dicts from items:
|
||||||
>>> dict(product) # create a dict from all populated values
|
>>> dict(product) # create a dict from all populated values
|
||||||
{'price': 1000, 'name': 'Desktop PC'}
|
{'price': 1000, 'name': 'Desktop PC'}
|
||||||
|
|
||||||
Creating items from dicts:
|
Creating items from dicts:
|
||||||
|
|
||||||
.. code-block:: pycon
|
|
||||||
|
|
||||||
>>> Product({"name": "Laptop PC", "price": 1500})
|
>>> Product({"name": "Laptop PC", "price": 1500})
|
||||||
{'name': 'Laptop PC', 'price': 1500}
|
Product(price=1500, name='Laptop PC')
|
||||||
|
|
||||||
>>> Product({"name": "Laptop PC", "lala": 1500}) # warning: unknown field in dict
|
>>> Product({"name": "Laptop PC", "lala": 1500}) # warning: unknown field in dict
|
||||||
Traceback (most recent call last):
|
Traceback (most recent call last):
|
||||||
|
|
|
||||||
|
|
@ -96,13 +96,9 @@ Request serialization
|
||||||
---------------------
|
---------------------
|
||||||
|
|
||||||
For persistence to work, :class:`~scrapy.Request` objects must be
|
For persistence to work, :class:`~scrapy.Request` objects must be
|
||||||
serializable with :mod:`pickle`, except for the :ref:`callback
|
serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
|
||||||
<callbacks>` and :ref:`errback
|
values passed to their ``__init__`` method, which must be methods of the
|
||||||
<errbacks>` values passed to their ``__init__``
|
running :class:`~scrapy.Spider` class.
|
||||||
method, which must be methods of the running :class:`~scrapy.Spider` class.
|
|
||||||
|
|
||||||
Requests that cannot be serialized are kept in memory only: they are still
|
|
||||||
sent, but they are lost when the crawl is paused.
|
|
||||||
|
|
||||||
If you wish to log the requests that couldn't be serialized, you can set the
|
If you wish to log the requests that couldn't be serialized, you can set the
|
||||||
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
|
:setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
|
||||||
|
|
@ -155,8 +151,8 @@ Where:
|
||||||
- :class:`~scrapy.pqueues.ScrapyPriorityQueue` creates the ``{priority}{s?}``
|
- :class:`~scrapy.pqueues.ScrapyPriorityQueue` creates the ``{priority}{s?}``
|
||||||
directories.
|
directories.
|
||||||
|
|
||||||
- :class:`scrapy.squeues.PickleFifoDiskQueue`, a subclass of
|
- :class:`scrapy.squeues.PickleLifoDiskQueue`, a subclass of
|
||||||
:class:`queuelib.FifoDiskQueue` that uses :mod:`pickle` to serialize
|
:class:`queuelib.LifoDiskQueue` that uses :mod:`pickle` to serialize
|
||||||
:class:`dict` representations of :class:`scrapy.Request` objects, creates
|
:class:`dict` representations of :class:`scrapy.Request` objects, creates
|
||||||
the ``info.json`` and ``q{00000}`` files.
|
the ``info.json`` and ``q{00000}`` files.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -62,9 +62,9 @@ Debugging memory leaks with ``trackref``
|
||||||
|
|
||||||
.. skip: start
|
.. skip: start
|
||||||
|
|
||||||
:mod:`scrapy.utils.trackref` is a module provided by Scrapy to debug the most
|
:mod:`trackref` is a module provided by Scrapy to debug the most common cases of
|
||||||
common cases of memory leaks. It basically tracks the references to all live
|
memory leaks. It basically tracks the references to all live Request,
|
||||||
Request, Response, Item, Spider and Selector objects.
|
Response, Item, Spider and Selector objects.
|
||||||
|
|
||||||
You can enter the telnet console and inspect how many objects (of the classes
|
You can enter the telnet console and inspect how many objects (of the classes
|
||||||
mentioned above) are currently alive using the ``prefs()`` function which is an
|
mentioned above) are currently alive using the ``prefs()`` function which is an
|
||||||
|
|
@ -93,7 +93,7 @@ You can get the oldest object of each class using the
|
||||||
Which objects are tracked?
|
Which objects are tracked?
|
||||||
--------------------------
|
--------------------------
|
||||||
|
|
||||||
The objects tracked by ``trackref`` are all from these classes (and all its
|
The objects tracked by ``trackrefs`` are all from these classes (and all its
|
||||||
subclasses):
|
subclasses):
|
||||||
|
|
||||||
* :class:`scrapy.Request`
|
* :class:`scrapy.Request`
|
||||||
|
|
@ -106,15 +106,10 @@ A real example
|
||||||
--------------
|
--------------
|
||||||
|
|
||||||
Let's see a concrete example of a hypothetical case of memory leaks.
|
Let's see a concrete example of a hypothetical case of memory leaks.
|
||||||
Suppose we have some spider with a line similar to this one:
|
Suppose we have some spider with a line similar to this one::
|
||||||
|
|
||||||
.. code-block:: python
|
return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}",
|
||||||
|
callback=self.parse, cb_kwargs={'referer': response})
|
||||||
return Request(
|
|
||||||
f"http://www.somenastyspider.com/product.php?pid={product_id}",
|
|
||||||
callback=self.parse,
|
|
||||||
cb_kwargs={"referer": response},
|
|
||||||
)
|
|
||||||
|
|
||||||
That line is passing a response reference inside a request which effectively
|
That line is passing a response reference inside a request which effectively
|
||||||
ties the response lifetime to the requests' one, and that would definitely
|
ties the response lifetime to the requests' one, and that would definitely
|
||||||
|
|
@ -169,7 +164,7 @@ Too many spiders?
|
||||||
-----------------
|
-----------------
|
||||||
|
|
||||||
If your project has too many spiders executed in parallel,
|
If your project has too many spiders executed in parallel,
|
||||||
the output of ``prefs()`` can be difficult to read.
|
the output of :func:`prefs` can be difficult to read.
|
||||||
For this reason, that function has a ``ignore`` argument which can be used to
|
For this reason, that function has a ``ignore`` argument which can be used to
|
||||||
ignore a particular class (and all its subclasses). For
|
ignore a particular class (and all its subclasses). For
|
||||||
example, this won't show any live references to spiders:
|
example, this won't show any live references to spiders:
|
||||||
|
|
@ -187,13 +182,30 @@ scrapy.utils.trackref module
|
||||||
|
|
||||||
Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
|
Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
|
||||||
|
|
||||||
.. autoclass:: object_ref
|
.. class:: object_ref
|
||||||
|
|
||||||
.. autofunction:: print_live_refs(ignore=NoneType)
|
Inherit from this class if you want to track live
|
||||||
|
instances with the ``trackref`` module.
|
||||||
|
|
||||||
.. autofunction:: get_oldest
|
.. function:: print_live_refs(class_name, ignore=NoneType)
|
||||||
|
|
||||||
.. autofunction:: iter_all
|
Print a report of live references, grouped by class name.
|
||||||
|
|
||||||
|
:param ignore: if given, all objects from the specified class (or tuple of
|
||||||
|
classes) will be ignored.
|
||||||
|
:type ignore: type or tuple
|
||||||
|
|
||||||
|
.. function:: get_oldest(class_name)
|
||||||
|
|
||||||
|
Return the oldest object alive with the given class name, or ``None`` if
|
||||||
|
none is found. Use :func:`print_live_refs` first to get a list of all
|
||||||
|
tracked live objects per class name.
|
||||||
|
|
||||||
|
.. function:: iter_all(class_name)
|
||||||
|
|
||||||
|
Return an iterator over all objects alive with the given class name, or
|
||||||
|
``None`` if none is found. Use :func:`print_live_refs` first to get a list
|
||||||
|
of all tracked live objects per class name.
|
||||||
|
|
||||||
.. skip: end
|
.. skip: end
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -36,9 +36,7 @@ Link extractor reference
|
||||||
|
|
||||||
The link extractor class is
|
The link extractor class is
|
||||||
:class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it
|
:class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it
|
||||||
can also be imported as ``scrapy.linkextractors.LinkExtractor``:
|
can also be imported as ``scrapy.linkextractors.LinkExtractor``::
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
from scrapy.linkextractors import LinkExtractor
|
from scrapy.linkextractors import LinkExtractor
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -76,7 +76,7 @@ data that will be assigned to the ``name`` field later.
|
||||||
|
|
||||||
Afterwards, similar calls are used for ``price`` and ``stock`` fields
|
Afterwards, similar calls are used for ``price`` and ``stock`` fields
|
||||||
(the latter using a CSS selector with the :meth:`~ItemLoader.add_css` method),
|
(the latter using a CSS selector with the :meth:`~ItemLoader.add_css` method),
|
||||||
and finally the ``last_updated`` field is populated directly with a literal value
|
and finally the ``last_update`` field is populated directly with a literal value
|
||||||
(``today``) using a different method: :meth:`~ItemLoader.add_value`.
|
(``today``) using a different method: :meth:`~ItemLoader.add_value`.
|
||||||
|
|
||||||
Finally, when all data is collected, the :meth:`ItemLoader.load_item` method is
|
Finally, when all data is collected, the :meth:`ItemLoader.load_item` method is
|
||||||
|
|
@ -264,7 +264,7 @@ metadata. Here is an example:
|
||||||
>>> il.add_value("name", ["Welcome to my", "<strong>website</strong>"])
|
>>> il.add_value("name", ["Welcome to my", "<strong>website</strong>"])
|
||||||
>>> il.add_value("price", ["€", "<span>1000</span>"])
|
>>> il.add_value("price", ["€", "<span>1000</span>"])
|
||||||
>>> il.load_item()
|
>>> il.load_item()
|
||||||
Product(name='Welcome to my website', price='1000')
|
{'name': 'Welcome to my website', 'price': '1000'}
|
||||||
|
|
||||||
.. skip: end
|
.. skip: end
|
||||||
|
|
||||||
|
|
@ -273,8 +273,8 @@ The precedence order, for both input and output processors, is as follows:
|
||||||
1. Item Loader field-specific attributes: ``field_in`` and ``field_out`` (most
|
1. Item Loader field-specific attributes: ``field_in`` and ``field_out`` (most
|
||||||
precedence)
|
precedence)
|
||||||
2. Field metadata (``input_processor`` and ``output_processor`` key)
|
2. Field metadata (``input_processor`` and ``output_processor`` key)
|
||||||
3. Item Loader defaults: :attr:`ItemLoader.default_input_processor` and
|
3. Item Loader defaults: :meth:`ItemLoader.default_input_processor` and
|
||||||
:attr:`ItemLoader.default_output_processor` (least precedence)
|
:meth:`ItemLoader.default_output_processor` (least precedence)
|
||||||
|
|
||||||
See also: :ref:`topics-loaders-extending`.
|
See also: :ref:`topics-loaders-extending`.
|
||||||
|
|
||||||
|
|
@ -323,8 +323,8 @@ There are several ways to modify Item Loader context values:
|
||||||
loader = ItemLoader(product, unit="cm")
|
loader = ItemLoader(product, unit="cm")
|
||||||
|
|
||||||
3. On Item Loader declaration, for those input/output processors that support
|
3. On Item Loader declaration, for those input/output processors that support
|
||||||
instantiating them with an Item Loader context.
|
instantiating them with an Item Loader context. :class:`~processor.MapCompose` is one of
|
||||||
:class:`~itemloaders.processors.MapCompose` is one of them:
|
them:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
|
|
@ -350,9 +350,7 @@ When parsing related values from a subsection of a document, it can be
|
||||||
useful to create nested loaders. Imagine you're extracting details from
|
useful to create nested loaders. Imagine you're extracting details from
|
||||||
a footer of a page that looks something like:
|
a footer of a page that looks something like:
|
||||||
|
|
||||||
Example:
|
Example::
|
||||||
|
|
||||||
.. code-block:: html
|
|
||||||
|
|
||||||
<footer>
|
<footer>
|
||||||
<a class="social" href="https://facebook.com/whatever">Like Us</a>
|
<a class="social" href="https://facebook.com/whatever">Like Us</a>
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,11 @@
|
||||||
Logging
|
Logging
|
||||||
=======
|
=======
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
:mod:`scrapy.log` has been deprecated alongside its functions in favor of
|
||||||
|
explicit calls to the Python standard logging. Keep reading to learn more
|
||||||
|
about the new logging system.
|
||||||
|
|
||||||
Scrapy uses :mod:`logging` for event logging. We'll
|
Scrapy uses :mod:`logging` for event logging. We'll
|
||||||
provide some simple examples to get you started, but for more advanced
|
provide some simple examples to get you started, but for more advanced
|
||||||
use-cases it's strongly suggested to read thoroughly its documentation.
|
use-cases it's strongly suggested to read thoroughly its documentation.
|
||||||
|
|
|
||||||
|
|
@ -41,10 +41,11 @@ this:
|
||||||
2. The item is returned from the spider and goes to the item pipeline.
|
2. The item is returned from the spider and goes to the item pipeline.
|
||||||
|
|
||||||
3. When the item reaches the :class:`FilesPipeline`, the URLs in the
|
3. When the item reaches the :class:`FilesPipeline`, the URLs in the
|
||||||
``file_urls`` field are downloaded using the standard Scrapy downloader
|
``file_urls`` field are scheduled for download using the standard
|
||||||
(which means the downloader middlewares are used, but the spider middlewares
|
Scrapy scheduler and downloader (which means the scheduler and downloader
|
||||||
aren't). The item remains "locked" at that particular pipeline stage until
|
middlewares are reused), but with a higher priority, processing them before other
|
||||||
the files have finished downloading (or failed for some reason).
|
pages are scraped. The item remains "locked" at that particular pipeline stage
|
||||||
|
until the files have finish downloading (or fail for some reason).
|
||||||
|
|
||||||
4. When the files are downloaded, another field (``files``) will be populated
|
4. When the files are downloaded, another field (``files``) will be populated
|
||||||
with the results. This field will contain a list of dicts with information
|
with the results. This field will contain a list of dicts with information
|
||||||
|
|
@ -60,8 +61,6 @@ this:
|
||||||
Using the Images Pipeline
|
Using the Images Pipeline
|
||||||
=========================
|
=========================
|
||||||
|
|
||||||
.. note:: Requires the :ref:`images <extras>` extra.
|
|
||||||
|
|
||||||
Using the :class:`ImagesPipeline` is a lot like using the :class:`FilesPipeline`,
|
Using the :class:`ImagesPipeline` is a lot like using the :class:`FilesPipeline`,
|
||||||
except the default field names used are different: you use ``image_urls`` for
|
except the default field names used are different: you use ``image_urls`` for
|
||||||
the image URLs of an item and it will populate an ``images`` field for the information
|
the image URLs of an item and it will populate an ``images`` field for the information
|
||||||
|
|
@ -71,6 +70,12 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
|
||||||
can configure some extra functions like generating thumbnails and filtering
|
can configure some extra functions like generating thumbnails and filtering
|
||||||
the images based on their size.
|
the images based on their size.
|
||||||
|
|
||||||
|
The Images Pipeline requires Pillow_ 8.3.2 or greater. It is used for
|
||||||
|
thumbnailing and normalizing images to JPEG/RGB format.
|
||||||
|
|
||||||
|
.. _Pillow: https://github.com/python-pillow/Pillow
|
||||||
|
|
||||||
|
|
||||||
.. _topics-media-pipeline-enabling:
|
.. _topics-media-pipeline-enabling:
|
||||||
|
|
||||||
Enabling your Media Pipeline
|
Enabling your Media Pipeline
|
||||||
|
|
@ -227,13 +232,12 @@ set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
|
||||||
Amazon S3 storage
|
Amazon S3 storage
|
||||||
-----------------
|
-----------------
|
||||||
|
|
||||||
.. note:: Requires the :ref:`s3 <extras>` extra.
|
|
||||||
|
|
||||||
.. setting:: FILES_STORE_S3_ACL
|
.. setting:: FILES_STORE_S3_ACL
|
||||||
.. setting:: IMAGES_STORE_S3_ACL
|
.. setting:: IMAGES_STORE_S3_ACL
|
||||||
|
|
||||||
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3
|
If botocore_ >= 1.13.45 is installed, :setting:`FILES_STORE` and
|
||||||
bucket. Scrapy will automatically upload the files to the bucket.
|
:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
|
||||||
|
automatically upload the files to the bucket.
|
||||||
|
|
||||||
For example, this is a valid :setting:`IMAGES_STORE` value:
|
For example, this is a valid :setting:`IMAGES_STORE` value:
|
||||||
|
|
||||||
|
|
@ -268,9 +272,7 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
|
||||||
AWS_USE_SSL = False # or True (None by default)
|
AWS_USE_SSL = False # or True (None by default)
|
||||||
AWS_VERIFY = False # or True (None by default)
|
AWS_VERIFY = False # or True (None by default)
|
||||||
|
|
||||||
To reuse connections for as many files as you check or upload in parallel, set
|
.. _botocore: https://github.com/boto/botocore
|
||||||
:setting:`AWS_MAX_POOL_CONNECTIONS` accordingly.
|
|
||||||
|
|
||||||
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
|
.. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
|
||||||
.. _Minio: https://github.com/minio/minio
|
.. _Minio: https://github.com/minio/minio
|
||||||
.. _Zenko CloudServer: https://www.zenko.io/cloudserver/
|
.. _Zenko CloudServer: https://www.zenko.io/cloudserver/
|
||||||
|
|
@ -281,13 +283,13 @@ To reuse connections for as many files as you check or upload in parallel, set
|
||||||
Google Cloud Storage
|
Google Cloud Storage
|
||||||
---------------------
|
---------------------
|
||||||
|
|
||||||
.. note:: Requires the :ref:`gcs <extras>` extra.
|
|
||||||
|
|
||||||
.. setting:: FILES_STORE_GCS_ACL
|
.. setting:: FILES_STORE_GCS_ACL
|
||||||
.. setting:: IMAGES_STORE_GCS_ACL
|
.. setting:: IMAGES_STORE_GCS_ACL
|
||||||
|
|
||||||
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud
|
:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage
|
||||||
Storage bucket. Scrapy will automatically upload the files to the bucket.
|
bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ )
|
||||||
|
|
||||||
|
.. _google-cloud-storage: https://docs.cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
|
||||||
|
|
||||||
For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings:
|
For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings:
|
||||||
|
|
||||||
|
|
@ -369,12 +371,11 @@ For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
|
||||||
If you need something more complex and want to override the custom pipeline
|
If you need something more complex and want to override the custom pipeline
|
||||||
behaviour, see :ref:`topics-media-pipeline-override`.
|
behaviour, see :ref:`topics-media-pipeline-override`.
|
||||||
|
|
||||||
If you have multiple image pipelines inheriting from :class:`ImagesPipeline`
|
If you have multiple image pipelines inheriting from ImagePipeline and you want
|
||||||
and you want to have different settings in different pipelines you can set
|
to have different settings in different pipelines you can set setting keys
|
||||||
setting keys preceded with uppercase name of your pipeline class. E.g. if your
|
preceded with uppercase name of your pipeline class. E.g. if your pipeline is
|
||||||
pipeline is called ``MyPipeline`` and you want to have custom
|
called MyPipeline and you want to have custom IMAGES_URLS_FIELD you define
|
||||||
:setting:`IMAGES_URLS_FIELD` you define setting
|
setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
|
||||||
``MYPIPELINE_IMAGES_URLS_FIELD`` and your custom settings will be used.
|
|
||||||
|
|
||||||
|
|
||||||
Additional features
|
Additional features
|
||||||
|
|
@ -469,9 +470,7 @@ When using the Images Pipeline, you can drop images which are too small, by
|
||||||
specifying the minimum allowed size in the :setting:`IMAGES_MIN_HEIGHT` and
|
specifying the minimum allowed size in the :setting:`IMAGES_MIN_HEIGHT` and
|
||||||
:setting:`IMAGES_MIN_WIDTH` settings.
|
:setting:`IMAGES_MIN_WIDTH` settings.
|
||||||
|
|
||||||
For example:
|
For example::
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
IMAGES_MIN_HEIGHT = 110
|
IMAGES_MIN_HEIGHT = 110
|
||||||
IMAGES_MIN_WIDTH = 110
|
IMAGES_MIN_WIDTH = 110
|
||||||
|
|
@ -494,9 +493,7 @@ Allowing redirections
|
||||||
By default media pipelines ignore redirects, i.e. an HTTP redirection
|
By default media pipelines ignore redirects, i.e. an HTTP redirection
|
||||||
to a media file URL request will mean the media download is considered failed.
|
to a media file URL request will mean the media download is considered failed.
|
||||||
|
|
||||||
To handle media redirections, set this setting to ``True``:
|
To handle media redirections, set this setting to ``True``::
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
MEDIA_ALLOW_REDIRECTS = True
|
MEDIA_ALLOW_REDIRECTS = True
|
||||||
|
|
||||||
|
|
@ -550,9 +547,10 @@ See here the methods that you can override in your custom Files Pipeline:
|
||||||
|
|
||||||
.. method:: FilesPipeline.get_media_requests(item, info)
|
.. method:: FilesPipeline.get_media_requests(item, info)
|
||||||
|
|
||||||
As seen on the workflow, the pipeline will get the requests for the files
|
As seen on the workflow, the pipeline will get the URLs of the images to
|
||||||
to download from the item by calling this method. You can override it to
|
download from the item. In order to do this, you can override the
|
||||||
change what requests are returned:
|
:meth:`~get_media_requests` method and return a Request for each
|
||||||
|
file URL:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
|
|
@ -592,9 +590,8 @@ See here the methods that you can override in your custom Files Pipeline:
|
||||||
* ``downloaded`` - file was downloaded.
|
* ``downloaded`` - file was downloaded.
|
||||||
* ``uptodate`` - file was not downloaded, as it was downloaded recently,
|
* ``uptodate`` - file was not downloaded, as it was downloaded recently,
|
||||||
according to the file expiration policy.
|
according to the file expiration policy.
|
||||||
* ``cached`` - file was taken from a cache (the response has a
|
* ``cached`` - file was already scheduled for download, by another item
|
||||||
``"cached"`` flag, e.g. from
|
sharing the same file.
|
||||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
|
|
||||||
|
|
||||||
The list of tuples received by :meth:`~item_completed` is
|
The list of tuples received by :meth:`~item_completed` is
|
||||||
guaranteed to retain the same order of the requests returned from the
|
guaranteed to retain the same order of the requests returned from the
|
||||||
|
|
@ -621,6 +618,9 @@ See here the methods that you can override in your custom Files Pipeline:
|
||||||
(False, Failure(...)),
|
(False, Failure(...)),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
By default the :meth:`get_media_requests` method returns ``None`` which
|
||||||
|
means there are no files to download for the item.
|
||||||
|
|
||||||
.. method:: FilesPipeline.item_completed(results, item, info)
|
.. method:: FilesPipeline.item_completed(results, item, info)
|
||||||
|
|
||||||
The :meth:`FilesPipeline.item_completed` method called when all file
|
The :meth:`FilesPipeline.item_completed` method called when all file
|
||||||
|
|
|
||||||
|
|
@ -17,10 +17,8 @@ Run Scrapy from a script
|
||||||
You can use the :ref:`API <topics-api>` to run Scrapy from a script, instead of
|
You can use the :ref:`API <topics-api>` to run Scrapy from a script, instead of
|
||||||
the typical way of running Scrapy via ``scrapy crawl``.
|
the typical way of running Scrapy via ``scrapy crawl``.
|
||||||
|
|
||||||
Remember that Scrapy requires a Twisted reactor or (with
|
Remember that Scrapy is built on top of the Twisted
|
||||||
:setting:`TWISTED_REACTOR_ENABLED` set to ``False``) an asyncio event loop, so
|
asynchronous networking library, so you need to run it inside the Twisted reactor.
|
||||||
you need to run one of those in your script for it to work (helpers described
|
|
||||||
below can do it for you).
|
|
||||||
|
|
||||||
The first utility you can use to run your spiders is
|
The first utility you can use to run your spiders is
|
||||||
:class:`scrapy.crawler.AsyncCrawlerProcess` or
|
:class:`scrapy.crawler.AsyncCrawlerProcess` or
|
||||||
|
|
@ -247,110 +245,6 @@ Using :func:`asyncio.run` with :class:`~scrapy.crawler.AsyncCrawlerRunner`:
|
||||||
|
|
||||||
asyncio.run(main())
|
asyncio.run(main())
|
||||||
|
|
||||||
.. _run-spiders-in-apps:
|
|
||||||
|
|
||||||
Running spiders inside existing applications
|
|
||||||
============================================
|
|
||||||
|
|
||||||
You may want to run Scrapy spiders inside an existing application. In simple
|
|
||||||
cases (e.g. task queues that spawn a process for every task, or applications
|
|
||||||
that can execute tasks synchronously in the same process) you can use the same
|
|
||||||
approach as for standalone scripts (see :ref:`run-from-script`). More complex
|
|
||||||
cases, e.g. asynchronous web applications, have additional caveats and
|
|
||||||
limitations.
|
|
||||||
|
|
||||||
If the application runs its own Twisted reactor, you can use
|
|
||||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` or
|
|
||||||
:class:`~scrapy.crawler.CrawlerRunner` to run spiders using this reactor, see
|
|
||||||
:ref:`run-from-script` for examples.
|
|
||||||
|
|
||||||
If the application doesn't run a Twisted reactor or an asyncio event loop (for
|
|
||||||
example, a Django web app deployed with a WSGI server such as uWSGI), you can
|
|
||||||
use :class:`~scrapy.crawler.AsyncCrawlerProcess` with
|
|
||||||
:setting:`TWISTED_REACTOR_ENABLED` set to ``False``, so that Scrapy starts and
|
|
||||||
stops an asyncio event loop for every spider run:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
import scrapy
|
|
||||||
from django.http import HttpResponse
|
|
||||||
from scrapy.crawler import AsyncCrawlerProcess
|
|
||||||
|
|
||||||
|
|
||||||
class MySpider(scrapy.Spider):
|
|
||||||
# Your spider definition
|
|
||||||
...
|
|
||||||
|
|
||||||
|
|
||||||
def crawl_view(request):
|
|
||||||
process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
|
|
||||||
process.crawl(MySpider)
|
|
||||||
process.start() # returns when the spider finishes
|
|
||||||
return HttpResponse("Crawling finished")
|
|
||||||
|
|
||||||
If the application runs its own asyncio event loop (for example, a Django web
|
|
||||||
app deployed with an ASGI server such as uvicorn), you can use
|
|
||||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` with
|
|
||||||
:setting:`TWISTED_REACTOR_ENABLED` set to ``False``, so that Scrapy uses the
|
|
||||||
existing event loop:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
import scrapy
|
|
||||||
from django.http import HttpResponse
|
|
||||||
from scrapy.crawler import AsyncCrawlerRunner
|
|
||||||
|
|
||||||
|
|
||||||
class MySpider(scrapy.Spider):
|
|
||||||
# Your spider definition
|
|
||||||
...
|
|
||||||
|
|
||||||
|
|
||||||
async def crawl_view(request):
|
|
||||||
runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
|
|
||||||
await runner.crawl(MySpider) # completes when the spider finishes
|
|
||||||
return HttpResponse("Crawling finished")
|
|
||||||
|
|
||||||
.. note:: Running Scrapy without a Twisted reactor is experimental and has
|
|
||||||
some limitations, described in :ref:`asyncio-without-reactor`.
|
|
||||||
|
|
||||||
.. _run-in-notebook:
|
|
||||||
|
|
||||||
Running spiders in Jupyter notebooks
|
|
||||||
====================================
|
|
||||||
|
|
||||||
You can run Scrapy spiders in Jupyter notebooks. You need to use
|
|
||||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` with
|
|
||||||
:setting:`TWISTED_REACTOR_ENABLED` set to ``False`` for this, so that Scrapy
|
|
||||||
uses the event loop provided by the notebook kernel. As
|
|
||||||
:class:`~scrapy.crawler.AsyncCrawlerRunner` doesn't configure logging, and you
|
|
||||||
most likely want to see the spider log in the notebook, you should call
|
|
||||||
:func:`scrapy.utils.log.configure_logging`. Here is a full example, which
|
|
||||||
supports rerunning both as a single cell and as separate cells:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
from scrapy import Spider
|
|
||||||
from scrapy.crawler import AsyncCrawlerRunner
|
|
||||||
from scrapy.utils.log import configure_logging
|
|
||||||
|
|
||||||
configure_logging()
|
|
||||||
|
|
||||||
|
|
||||||
class BooksSpider(Spider):
|
|
||||||
name = "books"
|
|
||||||
start_urls = ["https://books.toscrape.com"]
|
|
||||||
|
|
||||||
def parse(self, response):
|
|
||||||
for book in response.css("h3"):
|
|
||||||
yield {"title": book.css("a::attr(title)").get()}
|
|
||||||
|
|
||||||
|
|
||||||
runner = AsyncCrawlerRunner({"TWISTED_REACTOR_ENABLED": False})
|
|
||||||
await runner.crawl(BooksSpider)
|
|
||||||
|
|
||||||
.. note:: Running Scrapy without a Twisted reactor is experimental and has
|
|
||||||
some limitations, described in :ref:`asyncio-without-reactor`.
|
|
||||||
|
|
||||||
.. _run-multiple-spiders:
|
.. _run-multiple-spiders:
|
||||||
|
|
||||||
|
|
@ -533,7 +427,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
|
||||||
* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
|
* if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
|
||||||
directly
|
directly
|
||||||
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
|
* use a pool of rotating IPs. For example, the free `Tor project`_ or paid
|
||||||
services like `ProxyMesh`_.
|
services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
|
||||||
|
super proxy that you can attach your own proxies to.
|
||||||
* for HTTPS websites, if blocking appears related to TLS behavior, consider
|
* for HTTPS websites, if blocking appears related to TLS behavior, consider
|
||||||
adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and
|
adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and
|
||||||
:setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond
|
:setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond
|
||||||
|
|
@ -558,4 +453,5 @@ projects that detects common mistakes and anti-patterns.
|
||||||
.. _ProxyMesh: https://proxymesh.com/
|
.. _ProxyMesh: https://proxymesh.com/
|
||||||
.. _Common Crawl: https://commoncrawl.org/
|
.. _Common Crawl: https://commoncrawl.org/
|
||||||
.. _testspiders: https://github.com/scrapinghub/testspiders
|
.. _testspiders: https://github.com/scrapinghub/testspiders
|
||||||
|
.. _scrapoxy: https://scrapoxy.io/
|
||||||
.. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html
|
.. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html
|
||||||
|
|
|
||||||
|
|
@ -205,11 +205,10 @@ Request objects
|
||||||
Request metadata can also be accessed through the
|
Request metadata can also be accessed through the
|
||||||
:attr:`~scrapy.http.Response.meta` attribute of a response.
|
:attr:`~scrapy.http.Response.meta` attribute of a response.
|
||||||
|
|
||||||
To pass your own data from one spider callback to another, use
|
To pass data from one spider callback to another, consider using
|
||||||
:attr:`cb_kwargs` instead, see :ref:`callback-data`. However, request
|
:attr:`cb_kwargs` instead. However, request metadata may be the right
|
||||||
metadata may be the right choice in certain scenarios, such as to
|
choice in certain scenarios, such as to maintain some debugging data
|
||||||
maintain some debugging data across all follow-up requests (e.g. the
|
across all follow-up requests (e.g. the source URL).
|
||||||
source URL).
|
|
||||||
|
|
||||||
A common use of request metadata is to define request-specific
|
A common use of request metadata is to define request-specific
|
||||||
parameters for Scrapy components (extensions, middlewares, etc.). For
|
parameters for Scrapy components (extensions, middlewares, etc.). For
|
||||||
|
|
@ -239,9 +238,6 @@ Request objects
|
||||||
Also mind that the :meth:`copy` and :meth:`replace` request methods
|
Also mind that the :meth:`copy` and :meth:`replace` request methods
|
||||||
:doc:`shallow-copy <library/copy>` request metadata.
|
:doc:`shallow-copy <library/copy>` request metadata.
|
||||||
|
|
||||||
.. seealso:: :class:`~scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware`
|
|
||||||
for a built-in middleware that warns about this issue at run time.
|
|
||||||
|
|
||||||
.. autoattribute:: dont_filter
|
.. autoattribute:: dont_filter
|
||||||
|
|
||||||
.. autoattribute:: Request.attributes
|
.. autoattribute:: Request.attributes
|
||||||
|
|
@ -249,20 +245,18 @@ Request objects
|
||||||
.. method:: Request.copy()
|
.. method:: Request.copy()
|
||||||
|
|
||||||
Return a new Request which is a copy of this Request. See also:
|
Return a new Request which is a copy of this Request. See also:
|
||||||
:ref:`callback-data`.
|
:ref:`topics-request-response-ref-request-callback-arguments`.
|
||||||
|
|
||||||
.. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs, cls])
|
.. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs])
|
||||||
|
|
||||||
Return a Request object with the same members, except for those members
|
Return a Request object with the same members, except for those members
|
||||||
given new values by whichever keyword arguments are specified. The
|
given new values by whichever keyword arguments are specified. The
|
||||||
:attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` attributes are shallow
|
:attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` attributes are shallow
|
||||||
copied by default (unless new values are given as arguments). See also
|
copied by default (unless new values are given as arguments). See also
|
||||||
:ref:`callback-data`.
|
:ref:`topics-request-response-ref-request-callback-arguments`.
|
||||||
|
|
||||||
.. automethod:: from_curl
|
.. automethod:: from_curl
|
||||||
|
|
||||||
.. automethod:: to_curl
|
|
||||||
|
|
||||||
.. automethod:: to_dict
|
.. automethod:: to_dict
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -345,7 +339,159 @@ Other functions related to requests
|
||||||
|
|
||||||
.. autofunction:: scrapy.utils.request.request_from_dict
|
.. autofunction:: scrapy.utils.request.request_from_dict
|
||||||
|
|
||||||
.. autofunction:: scrapy.utils.httpobj.urlparse_cached
|
|
||||||
|
.. _topics-request-response-ref-request-callback-arguments:
|
||||||
|
|
||||||
|
Passing additional data to callback functions
|
||||||
|
---------------------------------------------
|
||||||
|
|
||||||
|
The callback of a request is a function that will be called when the response
|
||||||
|
of that request is downloaded. The callback function will be called with the
|
||||||
|
downloaded :class:`Response` object as its first argument.
|
||||||
|
|
||||||
|
Example:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
def parse_page1(self, response):
|
||||||
|
return scrapy.Request(
|
||||||
|
"http://www.example.com/some_page.html", callback=self.parse_page2
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_page2(self, response):
|
||||||
|
# this would log http://www.example.com/some_page.html
|
||||||
|
self.logger.info("Visited %s", response.url)
|
||||||
|
|
||||||
|
In some cases you may be interested in passing arguments to those callback
|
||||||
|
functions so you can receive the arguments later, in the second callback.
|
||||||
|
The following example shows how to achieve this by using the
|
||||||
|
:attr:`.Request.cb_kwargs` attribute:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
def parse(self, response):
|
||||||
|
request = scrapy.Request(
|
||||||
|
"http://www.example.com/index.html",
|
||||||
|
callback=self.parse_page2,
|
||||||
|
cb_kwargs=dict(main_url=response.url),
|
||||||
|
)
|
||||||
|
request.cb_kwargs["foo"] = "bar" # add more arguments for the callback
|
||||||
|
yield request
|
||||||
|
|
||||||
|
|
||||||
|
def parse_page2(self, response, main_url, foo):
|
||||||
|
yield dict(
|
||||||
|
main_url=main_url,
|
||||||
|
other_url=response.url,
|
||||||
|
foo=foo,
|
||||||
|
)
|
||||||
|
|
||||||
|
.. caution:: :attr:`.Request.cb_kwargs` was introduced in version ``1.7``.
|
||||||
|
Prior to that, using :attr:`.Request.meta` was recommended for passing
|
||||||
|
information around callbacks. After ``1.7``, :attr:`.Request.cb_kwargs`
|
||||||
|
became the preferred way for handling user information, leaving :attr:`.Request.meta`
|
||||||
|
for communication with components like middlewares and extensions.
|
||||||
|
|
||||||
|
.. _topics-request-response-ref-errbacks:
|
||||||
|
|
||||||
|
Using errbacks to catch exceptions in request processing
|
||||||
|
--------------------------------------------------------
|
||||||
|
|
||||||
|
The errback of a request is a function that will be called when an exception
|
||||||
|
is raise while processing it.
|
||||||
|
|
||||||
|
It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
|
||||||
|
be used to track connection establishment timeouts, DNS errors etc.
|
||||||
|
|
||||||
|
Here's an example spider logging all errors and catching some specific
|
||||||
|
errors if needed:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
import scrapy
|
||||||
|
|
||||||
|
from scrapy.spidermiddlewares.httperror import HttpError
|
||||||
|
from twisted.internet.error import DNSLookupError
|
||||||
|
from twisted.internet.error import TimeoutError, TCPTimedOutError
|
||||||
|
|
||||||
|
|
||||||
|
class ErrbackSpider(scrapy.Spider):
|
||||||
|
name = "errback_example"
|
||||||
|
start_urls = [
|
||||||
|
"http://www.httpbin.org/", # HTTP 200 expected
|
||||||
|
"http://www.httpbin.org/status/404", # Not found error
|
||||||
|
"http://www.httpbin.org/status/500", # server issue
|
||||||
|
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
|
||||||
|
"https://example.invalid/", # DNS error expected
|
||||||
|
]
|
||||||
|
|
||||||
|
async def start(self):
|
||||||
|
for u in self.start_urls:
|
||||||
|
yield scrapy.Request(
|
||||||
|
u,
|
||||||
|
callback=self.parse_httpbin,
|
||||||
|
errback=self.errback_httpbin,
|
||||||
|
dont_filter=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
def parse_httpbin(self, response):
|
||||||
|
self.logger.info("Got successful response from {}".format(response.url))
|
||||||
|
# do something useful here...
|
||||||
|
|
||||||
|
def errback_httpbin(self, failure):
|
||||||
|
# log all failures
|
||||||
|
self.logger.error(repr(failure))
|
||||||
|
|
||||||
|
# in case you want to do something special for some errors,
|
||||||
|
# you may need the failure's type:
|
||||||
|
|
||||||
|
if failure.check(HttpError):
|
||||||
|
# these exceptions come from HttpError spider middleware
|
||||||
|
# you can get the non-200 response
|
||||||
|
response = failure.value.response
|
||||||
|
self.logger.error("HttpError on %s", response.url)
|
||||||
|
|
||||||
|
elif failure.check(DNSLookupError):
|
||||||
|
# this is the original request
|
||||||
|
request = failure.request
|
||||||
|
self.logger.error("DNSLookupError on %s", request.url)
|
||||||
|
|
||||||
|
elif failure.check(TimeoutError, TCPTimedOutError):
|
||||||
|
request = failure.request
|
||||||
|
self.logger.error("TimeoutError on %s", request.url)
|
||||||
|
|
||||||
|
|
||||||
|
.. _errback-cb_kwargs:
|
||||||
|
|
||||||
|
Accessing additional data in errback functions
|
||||||
|
----------------------------------------------
|
||||||
|
|
||||||
|
In case of a failure to process the request, you may be interested in
|
||||||
|
accessing arguments to the callback functions so you can process further
|
||||||
|
based on the arguments in the errback. The following example shows how to
|
||||||
|
achieve this by using ``Failure.request.cb_kwargs``:
|
||||||
|
|
||||||
|
.. code-block:: python
|
||||||
|
|
||||||
|
def parse(self, response):
|
||||||
|
request = scrapy.Request(
|
||||||
|
"http://www.example.com/index.html",
|
||||||
|
callback=self.parse_page2,
|
||||||
|
errback=self.errback_page2,
|
||||||
|
cb_kwargs=dict(main_url=response.url),
|
||||||
|
)
|
||||||
|
yield request
|
||||||
|
|
||||||
|
|
||||||
|
def parse_page2(self, response, main_url):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def errback_page2(self, failure):
|
||||||
|
yield dict(
|
||||||
|
main_url=failure.request.cb_kwargs["main_url"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
.. _request-fingerprints:
|
.. _request-fingerprints:
|
||||||
|
|
@ -549,319 +695,6 @@ The following built-in Scrapy components have such restrictions:
|
||||||
45-character-long keys must be supported.
|
45-character-long keys must be supported.
|
||||||
|
|
||||||
|
|
||||||
.. _callbacks:
|
|
||||||
|
|
||||||
Callbacks
|
|
||||||
=========
|
|
||||||
|
|
||||||
A callback is a function that Scrapy calls with the :class:`Response` of a
|
|
||||||
:class:`~scrapy.Request` once that request has been downloaded, so that you can
|
|
||||||
extract data from that response and generate additional requests to continue
|
|
||||||
the crawl:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
from scrapy import Request, Spider
|
|
||||||
|
|
||||||
|
|
||||||
class BookSpider(Spider):
|
|
||||||
name = "books"
|
|
||||||
|
|
||||||
async def start(self):
|
|
||||||
yield Request("https://books.toscrape.com/", callback=self.parse_home)
|
|
||||||
|
|
||||||
def parse_home(self, response):
|
|
||||||
for url in response.css("h3 a::attr(href)").getall():
|
|
||||||
yield Request(response.urljoin(url), callback=self.parse_book)
|
|
||||||
|
|
||||||
def parse_book(self, response):
|
|
||||||
yield {"title": response.css("h1::text").get()}
|
|
||||||
|
|
||||||
Requests may also define an :ref:`errback <errbacks>`, which Scrapy calls
|
|
||||||
instead of the callback when an exception is raised while processing the
|
|
||||||
request or its response, e.g. a connection error or, by default, a non-2xx
|
|
||||||
response.
|
|
||||||
|
|
||||||
|
|
||||||
.. _callback-assignment:
|
|
||||||
|
|
||||||
Assigning a callback to a request
|
|
||||||
---------------------------------
|
|
||||||
|
|
||||||
To assign a callback to a request, use the ``callback`` parameter of
|
|
||||||
:class:`~scrapy.Request`, which sets the :attr:`.Request.callback` attribute:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
from scrapy import Request
|
|
||||||
|
|
||||||
|
|
||||||
def parse_home(response): ...
|
|
||||||
|
|
||||||
|
|
||||||
request = Request("https://books.toscrape.com/", callback=parse_home)
|
|
||||||
|
|
||||||
Requests with no callback, i.e. with :attr:`~scrapy.Request.callback` set to
|
|
||||||
``None``, are handled by the :meth:`~scrapy.Spider.parse` method of the spider:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
request = Request("https://books.toscrape.com/") # Handled by parse()
|
|
||||||
|
|
||||||
If a request is never meant to reach a spider callback, e.g. because a
|
|
||||||
:ref:`component <topics-components>` sends it and handles its response itself,
|
|
||||||
assign the special :func:`~scrapy.http.request.NO_CALLBACK` value to it
|
|
||||||
instead, so that :ref:`downloader middlewares <topics-downloader-middleware>`
|
|
||||||
can tell such requests apart.
|
|
||||||
|
|
||||||
While :attr:`~scrapy.Request.callback` only accepts callables, some spider
|
|
||||||
classes let you also define a callback by name: both :attr:`CrawlSpider.rules
|
|
||||||
<scrapy.spiders.CrawlSpider.rules>` and :attr:`SitemapSpider.sitemap_rules
|
|
||||||
<scrapy.spiders.SitemapSpider.sitemap_rules>` accept the name of a spider
|
|
||||||
method as a string.
|
|
||||||
|
|
||||||
|
|
||||||
.. _writing-callbacks:
|
|
||||||
|
|
||||||
Writing a callback
|
|
||||||
------------------
|
|
||||||
|
|
||||||
Any callable can be a callback, as long as it takes the response as its first
|
|
||||||
positional parameter, and any :ref:`additional callback data <callback-data>`
|
|
||||||
as keyword parameters. Spider methods are the most common choice, but plain
|
|
||||||
functions, lambda expressions and other callable objects work as well.
|
|
||||||
|
|
||||||
.. note:: If you enable :ref:`job persistence <topics-jobs>` through the
|
|
||||||
:setting:`JOBDIR` setting, callbacks must be methods of the running spider.
|
|
||||||
Requests with any other callback cannot be serialized, so they are kept in
|
|
||||||
memory only and lost when you pause the crawl. See
|
|
||||||
:ref:`request-serialization`.
|
|
||||||
|
|
||||||
A callback can be:
|
|
||||||
|
|
||||||
- A regular function:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
def parse(self, response):
|
|
||||||
return {"url": response.url}
|
|
||||||
|
|
||||||
- A generator function:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
def parse(self, response):
|
|
||||||
yield {"url": response.url}
|
|
||||||
|
|
||||||
- A coroutine function, i.e. defined with ``async def``:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
async def parse(self, response):
|
|
||||||
return {"url": response.url}
|
|
||||||
|
|
||||||
- An asynchronous generator function:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
async def parse(self, response):
|
|
||||||
yield {"url": response.url}
|
|
||||||
|
|
||||||
The last two allow using ``await``, ``async for`` and ``async with`` in your
|
|
||||||
callback. See :ref:`topics-coroutines`.
|
|
||||||
|
|
||||||
|
|
||||||
.. _callback-output:
|
|
||||||
|
|
||||||
Callback output
|
|
||||||
---------------
|
|
||||||
|
|
||||||
A callback may return or yield any of the following:
|
|
||||||
|
|
||||||
- ``None``, which does nothing.
|
|
||||||
|
|
||||||
Callbacks that produce no output at all, e.g. callbacks that only log
|
|
||||||
information about the response, are perfectly valid. ``None`` values within
|
|
||||||
an iterable of callback output are ignored as well.
|
|
||||||
|
|
||||||
- A :class:`~scrapy.Request` object, which Scrapy schedules, downloads and
|
|
||||||
eventually sends to its own callback.
|
|
||||||
|
|
||||||
- An :ref:`item object <topics-items>`, which Scrapy sends to the
|
|
||||||
:ref:`item pipelines <topics-item-pipeline>`.
|
|
||||||
|
|
||||||
Any object that is neither ``None`` nor a :class:`~scrapy.Request` object
|
|
||||||
is treated as an item.
|
|
||||||
|
|
||||||
- An iterable of any of the values above, e.g. a list or, more commonly, a
|
|
||||||
generator.
|
|
||||||
|
|
||||||
:term:`Asynchronous iterables <asynchronous iterable>`, e.g. an
|
|
||||||
:term:`asynchronous generator`, are also supported.
|
|
||||||
|
|
||||||
.. note:: When a callback *returns* an object, Scrapy iterates that object if
|
|
||||||
it supports iteration, except for :class:`dict`, :class:`~scrapy.Item`,
|
|
||||||
:class:`str` and :class:`bytes` objects, which are always handled as single
|
|
||||||
items.
|
|
||||||
|
|
||||||
.. note:: In a generator callback, a ``return`` statement with a value does not
|
|
||||||
produce any output, since such a value is not part of what the generator
|
|
||||||
yields. Scrapy logs a warning when it detects such a callback, see
|
|
||||||
:setting:`WARN_ON_GENERATOR_RETURN_VALUE`.
|
|
||||||
|
|
||||||
Before Scrapy acts on the output of a callback, that output goes through the
|
|
||||||
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
|
|
||||||
of your :ref:`spider middlewares <topics-spider-middleware>`, which may modify
|
|
||||||
it or drop part of it.
|
|
||||||
|
|
||||||
If a callback raises an exception, the :attr:`~scrapy.Request.errback` of the
|
|
||||||
request is *not* called. The exception goes through the
|
|
||||||
:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception`
|
|
||||||
method of your spider middlewares instead and, unless one of them handles it,
|
|
||||||
Scrapy logs it and sends the :signal:`spider_error` signal.
|
|
||||||
|
|
||||||
|
|
||||||
.. _callback-data:
|
|
||||||
.. _topics-request-response-ref-request-callback-arguments:
|
|
||||||
|
|
||||||
Passing additional data to callback functions
|
|
||||||
---------------------------------------------
|
|
||||||
|
|
||||||
In some cases you may be interested in passing data to a callback in addition
|
|
||||||
to the response, e.g. data extracted from the response that triggered the
|
|
||||||
request. The following example shows how to achieve this by using the
|
|
||||||
:attr:`.Request.cb_kwargs` attribute:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
from scrapy import Request
|
|
||||||
|
|
||||||
|
|
||||||
def parse(self, response):
|
|
||||||
request = Request(
|
|
||||||
"http://www.example.com/index.html",
|
|
||||||
callback=self.parse_page2,
|
|
||||||
cb_kwargs=dict(main_url=response.url),
|
|
||||||
)
|
|
||||||
request.cb_kwargs["foo"] = "bar" # add more arguments for the callback
|
|
||||||
yield request
|
|
||||||
|
|
||||||
|
|
||||||
def parse_page2(self, response, main_url, foo):
|
|
||||||
yield dict(
|
|
||||||
main_url=main_url,
|
|
||||||
other_url=response.url,
|
|
||||||
foo=foo,
|
|
||||||
)
|
|
||||||
|
|
||||||
:attr:`.Request.cb_kwargs` is the recommended way to pass your own data to a
|
|
||||||
callback. Use :attr:`.Request.meta` only for data aimed at :ref:`components
|
|
||||||
<topics-components>`, such as middlewares and extensions.
|
|
||||||
|
|
||||||
.. _errbacks:
|
|
||||||
.. _topics-request-response-ref-errbacks:
|
|
||||||
|
|
||||||
Errbacks
|
|
||||||
========
|
|
||||||
|
|
||||||
The errback of a request is a function that will be called when an exception
|
|
||||||
is raise while processing it.
|
|
||||||
|
|
||||||
It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
|
|
||||||
be used to track connection establishment timeouts, DNS errors etc.
|
|
||||||
|
|
||||||
Here's an example spider logging all errors and catching some specific
|
|
||||||
errors if needed:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
from scrapy import Request, Spider
|
|
||||||
from scrapy.spidermiddlewares.httperror import HttpError
|
|
||||||
from twisted.internet.error import DNSLookupError
|
|
||||||
from twisted.internet.error import TimeoutError, TCPTimedOutError
|
|
||||||
|
|
||||||
|
|
||||||
class ErrbackSpider(Spider):
|
|
||||||
name = "errback_example"
|
|
||||||
start_urls = [
|
|
||||||
"http://www.httpbin.org/", # HTTP 200 expected
|
|
||||||
"http://www.httpbin.org/status/404", # Not found error
|
|
||||||
"http://www.httpbin.org/status/500", # server issue
|
|
||||||
"http://www.httpbin.org:12345/", # non-responding host, timeout expected
|
|
||||||
"https://example.invalid/", # DNS error expected
|
|
||||||
]
|
|
||||||
|
|
||||||
async def start(self):
|
|
||||||
for u in self.start_urls:
|
|
||||||
yield Request(
|
|
||||||
u,
|
|
||||||
callback=self.parse_httpbin,
|
|
||||||
errback=self.errback_httpbin,
|
|
||||||
dont_filter=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
def parse_httpbin(self, response):
|
|
||||||
self.logger.info(f"Got successful response from {response.url}")
|
|
||||||
# do something useful here...
|
|
||||||
|
|
||||||
def errback_httpbin(self, failure):
|
|
||||||
# log all failures
|
|
||||||
self.logger.error(repr(failure))
|
|
||||||
|
|
||||||
# in case you want to do something special for some errors,
|
|
||||||
# you may need the failure's type:
|
|
||||||
|
|
||||||
if failure.check(HttpError):
|
|
||||||
# these exceptions come from HttpError spider middleware
|
|
||||||
# you can get the non-200 response
|
|
||||||
response = failure.value.response
|
|
||||||
self.logger.error("HttpError on %s", response.url)
|
|
||||||
|
|
||||||
elif failure.check(DNSLookupError):
|
|
||||||
# this is the original request
|
|
||||||
request = failure.request
|
|
||||||
self.logger.error("DNSLookupError on %s", request.url)
|
|
||||||
|
|
||||||
elif failure.check(TimeoutError, TCPTimedOutError):
|
|
||||||
request = failure.request
|
|
||||||
self.logger.error("TimeoutError on %s", request.url)
|
|
||||||
|
|
||||||
|
|
||||||
.. _errback-cb_kwargs:
|
|
||||||
|
|
||||||
Accessing additional data in errback functions
|
|
||||||
----------------------------------------------
|
|
||||||
|
|
||||||
In case of a failure to process the request, you may be interested in
|
|
||||||
accessing arguments to the callback functions so you can process further
|
|
||||||
based on the arguments in the errback. The following example shows how to
|
|
||||||
achieve this by using ``Failure.request.cb_kwargs``:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
from scrapy import Request
|
|
||||||
|
|
||||||
|
|
||||||
def parse(self, response):
|
|
||||||
request = Request(
|
|
||||||
"http://www.example.com/index.html",
|
|
||||||
callback=self.parse_page2,
|
|
||||||
errback=self.errback_page2,
|
|
||||||
cb_kwargs=dict(main_url=response.url),
|
|
||||||
)
|
|
||||||
yield request
|
|
||||||
|
|
||||||
|
|
||||||
def parse_page2(self, response, main_url):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def errback_page2(self, failure):
|
|
||||||
yield dict(
|
|
||||||
main_url=failure.request.cb_kwargs["main_url"],
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
.. _topics-request-meta:
|
.. _topics-request-meta:
|
||||||
|
|
||||||
Request.meta special keys
|
Request.meta special keys
|
||||||
|
|
@ -884,7 +717,6 @@ Those are:
|
||||||
* :reqmeta:`download_fail_on_dataloss`
|
* :reqmeta:`download_fail_on_dataloss`
|
||||||
* :reqmeta:`download_latency`
|
* :reqmeta:`download_latency`
|
||||||
* :reqmeta:`download_maxsize`
|
* :reqmeta:`download_maxsize`
|
||||||
* :reqmeta:`download_slot`
|
|
||||||
* :reqmeta:`download_warnsize`
|
* :reqmeta:`download_warnsize`
|
||||||
* :reqmeta:`download_timeout`
|
* :reqmeta:`download_timeout`
|
||||||
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
|
* ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
|
||||||
|
|
@ -1195,13 +1027,9 @@ Response objects
|
||||||
:meth:`~scrapy.http.headers.Headers.get` to return the last header value with
|
:meth:`~scrapy.http.headers.Headers.get` to return the last header value with
|
||||||
the specified name or :meth:`~scrapy.http.headers.Headers.getlist` to return
|
the specified name or :meth:`~scrapy.http.headers.Headers.getlist` to return
|
||||||
all header values with the specified name. For example, this call will give you
|
all header values with the specified name. For example, this call will give you
|
||||||
all cookies in the headers:
|
all cookies in the headers::
|
||||||
|
|
||||||
.. skip: next
|
response.headers.getlist('Set-Cookie')
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
response.headers.getlist("Set-Cookie")
|
|
||||||
|
|
||||||
.. attribute:: Response.body
|
.. attribute:: Response.body
|
||||||
|
|
||||||
|
|
@ -1257,7 +1085,7 @@ Response objects
|
||||||
.. attribute:: Response.flags
|
.. attribute:: Response.flags
|
||||||
|
|
||||||
A list that contains flags for this response. Flags are labels used for
|
A list that contains flags for this response. Flags are labels used for
|
||||||
tagging Responses. For example: ``'cached'``, ``'redirected'``', etc. And
|
tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And
|
||||||
they're shown on the string representation of the Response (``__str__()``
|
they're shown on the string representation of the Response (``__str__()``
|
||||||
method) which is used by the engine for logging.
|
method) which is used by the engine for logging.
|
||||||
|
|
||||||
|
|
@ -1272,8 +1100,8 @@ Response objects
|
||||||
|
|
||||||
The IP address of the server from which the Response originated.
|
The IP address of the server from which the Response originated.
|
||||||
|
|
||||||
This attribute is currently only populated by the HTTP download
|
This attribute is currently only populated by the HTTP 1.1 download
|
||||||
handlers, i.e. for ``http(s)`` responses. For other handlers,
|
handler, i.e. for ``http(s)`` responses. For other handlers,
|
||||||
:attr:`ip_address` is always ``None``.
|
:attr:`ip_address` is always ``None``.
|
||||||
|
|
||||||
.. attribute:: Response.protocol
|
.. attribute:: Response.protocol
|
||||||
|
|
@ -1291,7 +1119,7 @@ Response objects
|
||||||
|
|
||||||
Returns a new Response which is a copy of this Response.
|
Returns a new Response which is a copy of this Response.
|
||||||
|
|
||||||
.. method:: Response.replace([url, status, headers, body, request, flags, certificate, ip_address, protocol, cls])
|
.. method:: Response.replace([url, status, headers, body, request, flags, cls])
|
||||||
|
|
||||||
Returns a Response object with the same members, except for those members
|
Returns a Response object with the same members, except for those members
|
||||||
given new values by whichever keyword arguments are specified. The
|
given new values by whichever keyword arguments are specified. The
|
||||||
|
|
@ -1303,11 +1131,7 @@ Response objects
|
||||||
a possible relative url.
|
a possible relative url.
|
||||||
|
|
||||||
This is a wrapper over :func:`~urllib.parse.urljoin`, it's merely an alias for
|
This is a wrapper over :func:`~urllib.parse.urljoin`, it's merely an alias for
|
||||||
making this call:
|
making this call::
|
||||||
|
|
||||||
.. skip: next
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
urllib.parse.urljoin(response.url, url)
|
urllib.parse.urljoin(response.url, url)
|
||||||
|
|
||||||
|
|
@ -1396,31 +1220,21 @@ TextResponse objects
|
||||||
|
|
||||||
.. method:: TextResponse.jmespath(query)
|
.. method:: TextResponse.jmespath(query)
|
||||||
|
|
||||||
.. skip: start
|
A shortcut to ``TextResponse.selector.jmespath(query)``::
|
||||||
|
|
||||||
A shortcut to ``TextResponse.selector.jmespath(query)``:
|
response.jmespath('object.[*]')
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
response.jmespath("object.[*]")
|
|
||||||
|
|
||||||
.. method:: TextResponse.xpath(query)
|
.. method:: TextResponse.xpath(query)
|
||||||
|
|
||||||
A shortcut to ``TextResponse.selector.xpath(query)``:
|
A shortcut to ``TextResponse.selector.xpath(query)``::
|
||||||
|
|
||||||
.. code-block:: python
|
response.xpath('//p')
|
||||||
|
|
||||||
response.xpath("//p")
|
|
||||||
|
|
||||||
.. method:: TextResponse.css(query)
|
.. method:: TextResponse.css(query)
|
||||||
|
|
||||||
A shortcut to ``TextResponse.selector.css(query)``:
|
A shortcut to ``TextResponse.selector.css(query)``::
|
||||||
|
|
||||||
.. code-block:: python
|
response.css('p')
|
||||||
|
|
||||||
response.css("p")
|
|
||||||
|
|
||||||
.. skip: end
|
|
||||||
|
|
||||||
.. automethod:: TextResponse.follow
|
.. automethod:: TextResponse.follow
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -308,7 +308,7 @@ Examples:
|
||||||
|
|
||||||
* ``*::text`` selects all descendant text nodes of the current selector context:
|
* ``*::text`` selects all descendant text nodes of the current selector context:
|
||||||
|
|
||||||
.. skip: next
|
..skip: next
|
||||||
.. code-block:: pycon
|
.. code-block:: pycon
|
||||||
|
|
||||||
>>> response.css("#images *::text").getall()
|
>>> response.css("#images *::text").getall()
|
||||||
|
|
@ -634,7 +634,8 @@ Example:
|
||||||
.. code-block:: pycon
|
.. code-block:: pycon
|
||||||
|
|
||||||
>>> from scrapy import Selector
|
>>> from scrapy import Selector
|
||||||
>>> sel = Selector(text="""
|
>>> sel = Selector(
|
||||||
|
... text="""
|
||||||
... <ul class="list">
|
... <ul class="list">
|
||||||
... <li>1</li>
|
... <li>1</li>
|
||||||
... <li>2</li>
|
... <li>2</li>
|
||||||
|
|
@ -644,8 +645,8 @@ Example:
|
||||||
... <li>4</li>
|
... <li>4</li>
|
||||||
... <li>5</li>
|
... <li>5</li>
|
||||||
... <li>6</li>
|
... <li>6</li>
|
||||||
... </ul>""")
|
... </ul>"""
|
||||||
...
|
... )
|
||||||
>>> xp = lambda x: sel.xpath(x).getall()
|
>>> xp = lambda x: sel.xpath(x).getall()
|
||||||
|
|
||||||
This gets all first ``<li>`` elements under whatever it is its parent:
|
This gets all first ``<li>`` elements under whatever it is its parent:
|
||||||
|
|
@ -947,9 +948,11 @@ with groups of itemscopes and corresponding itemprops:
|
||||||
>>> sel = Selector(text=doc, type="html")
|
>>> sel = Selector(text=doc, type="html")
|
||||||
>>> for scope in sel.xpath("//div[@itemscope]"):
|
>>> for scope in sel.xpath("//div[@itemscope]"):
|
||||||
... print("current scope:", scope.xpath("@itemtype").getall())
|
... print("current scope:", scope.xpath("@itemtype").getall())
|
||||||
... props = scope.xpath("""
|
... props = scope.xpath(
|
||||||
|
... """
|
||||||
... set:difference(./descendant::*/@itemprop,
|
... set:difference(./descendant::*/@itemprop,
|
||||||
... .//*[@itemscope]/*/@itemprop)""")
|
... .//*[@itemscope]/*/@itemprop)"""
|
||||||
|
... )
|
||||||
... print(f" properties: {props.getall()}")
|
... print(f" properties: {props.getall()}")
|
||||||
... print("")
|
... print("")
|
||||||
...
|
...
|
||||||
|
|
|
||||||
|
|
@ -305,21 +305,10 @@ These settings cannot be :ref:`set from a spider <spider-settings>`.
|
||||||
|
|
||||||
These settings are:
|
These settings are:
|
||||||
|
|
||||||
- :setting:`ADDONS`
|
- :setting:`TWISTED_REACTOR_ENABLED`
|
||||||
- :setting:`COMMANDS_MODULE`
|
|
||||||
- :setting:`FORCE_CRAWLER_PROCESS`
|
|
||||||
- :setting:`SPIDER_LOADER_CLASS` and settings used by the corresponding
|
- :setting:`SPIDER_LOADER_CLASS` and settings used by the corresponding
|
||||||
spider loader class, e.g. :setting:`SPIDER_MODULES` and
|
spider loader class, e.g. :setting:`SPIDER_MODULES` and
|
||||||
:setting:`SPIDER_LOADER_WARN_ONLY` for the default spider loader class.
|
:setting:`SPIDER_LOADER_WARN_ONLY` for the default spider loader class.
|
||||||
- :setting:`TWISTED_REACTOR_ENABLED`
|
|
||||||
|
|
||||||
:setting:`ADDONS` is a special case: it can be set from a spider, but the
|
|
||||||
``update_pre_crawler_settings()`` method of :ref:`add-ons <topics-addons>`
|
|
||||||
enabled that way is not called.
|
|
||||||
|
|
||||||
:setting:`TWISTED_REACTOR` also acts as a pre-crawler setting when running a
|
|
||||||
:ref:`command that needs a CrawlerProcess <topics-commands-crawlerprocess>`,
|
|
||||||
since its project-level value determines the crawler process class.
|
|
||||||
|
|
||||||
.. _reactor-settings:
|
.. _reactor-settings:
|
||||||
|
|
||||||
|
|
@ -420,39 +409,6 @@ Default: ``{}``
|
||||||
A dict containing paths to the add-ons enabled in your project and their
|
A dict containing paths to the add-ons enabled in your project and their
|
||||||
priorities. For more information, see :ref:`topics-addons`.
|
priorities. For more information, see :ref:`topics-addons`.
|
||||||
|
|
||||||
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`, with a
|
|
||||||
caveat described in that section.
|
|
||||||
|
|
||||||
.. setting:: ASYNCIO_EVENT_LOOP
|
|
||||||
|
|
||||||
ASYNCIO_EVENT_LOOP
|
|
||||||
------------------
|
|
||||||
|
|
||||||
Default: ``None``
|
|
||||||
|
|
||||||
Import path of a given ``asyncio`` event loop class.
|
|
||||||
|
|
||||||
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) or when
|
|
||||||
:ref:`running Scrapy without a reactor <asyncio-without-reactor>` this setting
|
|
||||||
can be used to specify the
|
|
||||||
asyncio event loop to be used with it. Set the setting to the import path of the
|
|
||||||
desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
|
|
||||||
event loop will be used.
|
|
||||||
|
|
||||||
If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
|
|
||||||
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
|
|
||||||
class to be used.
|
|
||||||
|
|
||||||
Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
|
|
||||||
|
|
||||||
.. caution:: Please be aware that, when using a non-default event loop
|
|
||||||
(either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
|
|
||||||
:func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
|
|
||||||
:func:`asyncio.set_event_loop`, which will set the specified event loop
|
|
||||||
as the current loop for the current OS thread.
|
|
||||||
|
|
||||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
|
||||||
|
|
||||||
.. setting:: AWS_ACCESS_KEY_ID
|
.. setting:: AWS_ACCESS_KEY_ID
|
||||||
|
|
||||||
AWS_ACCESS_KEY_ID
|
AWS_ACCESS_KEY_ID
|
||||||
|
|
@ -463,44 +419,6 @@ Default: ``None``
|
||||||
The AWS access key used by code that requires access to `Amazon Web services`_,
|
The AWS access key used by code that requires access to `Amazon Web services`_,
|
||||||
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
|
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
|
||||||
|
|
||||||
.. setting:: AWS_ENDPOINT_URL
|
|
||||||
|
|
||||||
AWS_ENDPOINT_URL
|
|
||||||
----------------
|
|
||||||
|
|
||||||
Default: ``None``
|
|
||||||
|
|
||||||
Endpoint URL used for S3-like storage, for example Minio or s3.scality.
|
|
||||||
|
|
||||||
.. setting:: AWS_MAX_POOL_CONNECTIONS
|
|
||||||
|
|
||||||
AWS_MAX_POOL_CONNECTIONS
|
|
||||||
------------------------
|
|
||||||
|
|
||||||
.. versionadded:: VERSION
|
|
||||||
|
|
||||||
Default: ``None``
|
|
||||||
|
|
||||||
Maximum number of connections that AWS clients, such as those of the
|
|
||||||
:ref:`S3 feed storage backend <topics-feed-storage-s3>` and of the
|
|
||||||
:ref:`S3 media pipeline storage backend <media-pipelines-s3>`, keep in their
|
|
||||||
connection pool.
|
|
||||||
|
|
||||||
If ``None``, the value of :setting:`REACTOR_THREADPOOL_MAXSIZE` is used.
|
|
||||||
|
|
||||||
Values lower than the number of parallel AWS calls do not limit those calls, but
|
|
||||||
their connections are closed instead of reused, which hurts performance, and
|
|
||||||
``Connection pool is full, discarding connection`` warnings are logged.
|
|
||||||
|
|
||||||
.. setting:: AWS_REGION_NAME
|
|
||||||
|
|
||||||
AWS_REGION_NAME
|
|
||||||
---------------
|
|
||||||
|
|
||||||
Default: ``None``
|
|
||||||
|
|
||||||
The name of the region associated with the AWS client.
|
|
||||||
|
|
||||||
.. setting:: AWS_SECRET_ACCESS_KEY
|
.. setting:: AWS_SECRET_ACCESS_KEY
|
||||||
|
|
||||||
AWS_SECRET_ACCESS_KEY
|
AWS_SECRET_ACCESS_KEY
|
||||||
|
|
@ -524,6 +442,15 @@ such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using
|
||||||
|
|
||||||
.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
|
.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
|
||||||
|
|
||||||
|
.. setting:: AWS_ENDPOINT_URL
|
||||||
|
|
||||||
|
AWS_ENDPOINT_URL
|
||||||
|
----------------
|
||||||
|
|
||||||
|
Default: ``None``
|
||||||
|
|
||||||
|
Endpoint URL used for S3-like storage, for example Minio or s3.scality.
|
||||||
|
|
||||||
.. setting:: AWS_USE_SSL
|
.. setting:: AWS_USE_SSL
|
||||||
|
|
||||||
AWS_USE_SSL
|
AWS_USE_SSL
|
||||||
|
|
@ -544,6 +471,43 @@ Default: ``None``
|
||||||
Verify SSL connection between Scrapy and S3 or S3-like storage. By default
|
Verify SSL connection between Scrapy and S3 or S3-like storage. By default
|
||||||
SSL verification will occur.
|
SSL verification will occur.
|
||||||
|
|
||||||
|
.. setting:: AWS_REGION_NAME
|
||||||
|
|
||||||
|
AWS_REGION_NAME
|
||||||
|
---------------
|
||||||
|
|
||||||
|
Default: ``None``
|
||||||
|
|
||||||
|
The name of the region associated with the AWS client.
|
||||||
|
|
||||||
|
.. setting:: ASYNCIO_EVENT_LOOP
|
||||||
|
|
||||||
|
ASYNCIO_EVENT_LOOP
|
||||||
|
------------------
|
||||||
|
|
||||||
|
Default: ``None``
|
||||||
|
|
||||||
|
Import path of a given ``asyncio`` event loop class.
|
||||||
|
|
||||||
|
If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
|
||||||
|
asyncio event loop to be used with it. Set the setting to the import path of the
|
||||||
|
desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
|
||||||
|
event loop will be used.
|
||||||
|
|
||||||
|
If you are installing the asyncio reactor manually using the :func:`~scrapy.utils.reactor.install_reactor`
|
||||||
|
function, you can use the ``event_loop_path`` parameter to indicate the import path of the event loop
|
||||||
|
class to be used.
|
||||||
|
|
||||||
|
Note that the event loop class must inherit from :class:`asyncio.AbstractEventLoop`.
|
||||||
|
|
||||||
|
.. caution:: Please be aware that, when using a non-default event loop
|
||||||
|
(either defined via :setting:`ASYNCIO_EVENT_LOOP` or installed with
|
||||||
|
:func:`~scrapy.utils.reactor.install_reactor`), Scrapy will call
|
||||||
|
:func:`asyncio.set_event_loop`, which will set the specified event loop
|
||||||
|
as the current loop for the current OS thread.
|
||||||
|
|
||||||
|
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||||
|
|
||||||
.. setting:: BOT_NAME
|
.. setting:: BOT_NAME
|
||||||
|
|
||||||
BOT_NAME
|
BOT_NAME
|
||||||
|
|
@ -575,7 +539,7 @@ CONCURRENT_REQUESTS
|
||||||
Default: ``16``
|
Default: ``16``
|
||||||
|
|
||||||
The maximum number of concurrent (i.e. simultaneous) requests that will be
|
The maximum number of concurrent (i.e. simultaneous) requests that will be
|
||||||
performed by the Scrapy downloader. Use ``0`` for no limit.
|
performed by the Scrapy downloader.
|
||||||
|
|
||||||
.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
|
.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
|
||||||
|
|
||||||
|
|
@ -590,8 +554,6 @@ performed to any single domain.
|
||||||
See also: :ref:`topics-autothrottle` and its
|
See also: :ref:`topics-autothrottle` and its
|
||||||
:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option.
|
:setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option.
|
||||||
|
|
||||||
It is possible to change this setting per domain by using
|
|
||||||
:setting:`DOWNLOAD_SLOTS`.
|
|
||||||
|
|
||||||
.. setting:: DEFAULT_DROPITEM_LOG_LEVEL
|
.. setting:: DEFAULT_DROPITEM_LOG_LEVEL
|
||||||
|
|
||||||
|
|
@ -631,7 +593,7 @@ When writing an item pipeline, you can force a different log level by setting
|
||||||
DEFAULT_ITEM_CLASS
|
DEFAULT_ITEM_CLASS
|
||||||
------------------
|
------------------
|
||||||
|
|
||||||
Default: ``'scrapy.item.Item'``
|
Default: ``'scrapy.Item'``
|
||||||
|
|
||||||
The default class that will be used for instantiating items in the :ref:`the
|
The default class that will be used for instantiating items in the :ref:`the
|
||||||
Scrapy shell <topics-shell>`.
|
Scrapy shell <topics-shell>`.
|
||||||
|
|
@ -725,7 +687,7 @@ Whether to enable DNS in-memory cache.
|
||||||
:class:`~scrapy.resolver.CachingThreadedResolver` and
|
:class:`~scrapy.resolver.CachingThreadedResolver` and
|
||||||
:class:`~scrapy.resolver.CachingHostnameResolver`. It has no effect when
|
:class:`~scrapy.resolver.CachingHostnameResolver`. It has no effect when
|
||||||
:setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
|
:setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
|
||||||
either when :setting:`TWISTED_DNS_RESOLVER` is set to a different resolver.
|
either when :setting:`DNS_RESOLVER` is set to a different resolver.
|
||||||
|
|
||||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||||
|
|
||||||
|
|
@ -740,6 +702,25 @@ DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`.
|
||||||
|
|
||||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||||
|
|
||||||
|
.. setting:: TWISTED_DNS_RESOLVER
|
||||||
|
|
||||||
|
TWISTED_DNS_RESOLVER
|
||||||
|
--------------------
|
||||||
|
|
||||||
|
Default: ``'scrapy.resolver.CachingThreadedResolver'``
|
||||||
|
|
||||||
|
The class to be used by Twisted to resolve DNS names. The default
|
||||||
|
``scrapy.resolver.CachingThreadedResolver`` supports specifying a timeout for
|
||||||
|
DNS requests via the :setting:`DNS_TIMEOUT` setting, but works only with IPv4
|
||||||
|
addresses. Scrapy provides an alternative resolver,
|
||||||
|
``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
|
||||||
|
take the :setting:`DNS_TIMEOUT` setting into account.
|
||||||
|
|
||||||
|
.. note::
|
||||||
|
This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
||||||
|
|
||||||
|
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||||
|
|
||||||
.. setting:: DNS_TIMEOUT
|
.. setting:: DNS_TIMEOUT
|
||||||
|
|
||||||
DNS_TIMEOUT
|
DNS_TIMEOUT
|
||||||
|
|
@ -753,7 +734,7 @@ Timeout for processing of DNS queries in seconds. Float is supported.
|
||||||
This setting is only used by
|
This setting is only used by
|
||||||
:class:`~scrapy.resolver.CachingThreadedResolver`. It has no effect when
|
:class:`~scrapy.resolver.CachingThreadedResolver`. It has no effect when
|
||||||
:setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
|
:setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
|
||||||
either when :setting:`TWISTED_DNS_RESOLVER` is set to a different resolver.
|
either when :setting:`DNS_RESOLVER` is set to a different resolver.
|
||||||
|
|
||||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
||||||
|
|
||||||
|
|
@ -933,9 +914,7 @@ Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting
|
||||||
servers too hard.
|
servers too hard.
|
||||||
|
|
||||||
Decimal numbers are supported. For example, to send a maximum of 4 requests
|
Decimal numbers are supported. For example, to send a maximum of 4 requests
|
||||||
every 10 seconds:
|
every 10 seconds::
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
DOWNLOAD_DELAY = 2.5
|
DOWNLOAD_DELAY = 2.5
|
||||||
|
|
||||||
|
|
@ -957,8 +936,9 @@ desired.
|
||||||
|
|
||||||
This delay can be set per spider using :attr:`download_delay` spider attribute.
|
This delay can be set per spider using :attr:`download_delay` spider attribute.
|
||||||
|
|
||||||
It is possible to change this setting per domain by using
|
It is also possible to change this setting per domain, although it requires
|
||||||
:setting:`DOWNLOAD_SLOTS`.
|
non-trivial code. See the implementation of the :ref:`AutoThrottle
|
||||||
|
<topics-autothrottle>` extension for an example.
|
||||||
|
|
||||||
.. setting:: DOWNLOAD_BIND_ADDRESS
|
.. setting:: DOWNLOAD_BIND_ADDRESS
|
||||||
|
|
||||||
|
|
@ -1267,9 +1247,7 @@ the ``dont_filter`` parameter to ``True`` on the ``__init__`` method of a
|
||||||
specific :class:`~scrapy.Request` object that should not be filtered out.
|
specific :class:`~scrapy.Request` object that should not be filtered out.
|
||||||
|
|
||||||
A class assigned to :setting:`DUPEFILTER_CLASS` must implement the following
|
A class assigned to :setting:`DUPEFILTER_CLASS` must implement the following
|
||||||
interface:
|
interface::
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
class MyDupeFilter:
|
class MyDupeFilter:
|
||||||
|
|
||||||
|
|
@ -1352,7 +1330,6 @@ Default:
|
||||||
|
|
||||||
{
|
{
|
||||||
"scrapy.extensions.corestats.CoreStats": 0,
|
"scrapy.extensions.corestats.CoreStats": 0,
|
||||||
"scrapy.extensions.logcount.LogCount": 0,
|
|
||||||
"scrapy.extensions.telnet.TelnetConsole": 0,
|
"scrapy.extensions.telnet.TelnetConsole": 0,
|
||||||
"scrapy.extensions.memusage.MemoryUsage": 0,
|
"scrapy.extensions.memusage.MemoryUsage": 0,
|
||||||
"scrapy.extensions.memdebug.MemoryDebugger": 0,
|
"scrapy.extensions.memdebug.MemoryDebugger": 0,
|
||||||
|
|
@ -1375,8 +1352,6 @@ and the :ref:`list of available extensions <topics-extensions-ref>`.
|
||||||
FEED_TEMPDIR
|
FEED_TEMPDIR
|
||||||
------------
|
------------
|
||||||
|
|
||||||
Default: ``None``
|
|
||||||
|
|
||||||
The Feed Temp dir allows you to set a custom folder to save crawler
|
The Feed Temp dir allows you to set a custom folder to save crawler
|
||||||
temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
|
temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
|
||||||
:ref:`Amazon S3 <topics-feed-storage-s3>`.
|
:ref:`Amazon S3 <topics-feed-storage-s3>`.
|
||||||
|
|
@ -1386,8 +1361,6 @@ temporary files before uploading with :ref:`FTP feed storage <topics-feed-storag
|
||||||
FEED_STORAGE_GCS_ACL
|
FEED_STORAGE_GCS_ACL
|
||||||
--------------------
|
--------------------
|
||||||
|
|
||||||
Default: ``""``
|
|
||||||
|
|
||||||
The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
|
The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
|
||||||
For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://docs.cloud.google.com/storage/docs/access-control/lists>`_.
|
For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://docs.cloud.google.com/storage/docs/access-control/lists>`_.
|
||||||
|
|
||||||
|
|
@ -1399,25 +1372,18 @@ FORCE_CRAWLER_PROCESS
|
||||||
Default: ``False``
|
Default: ``False``
|
||||||
|
|
||||||
If ``False``, :ref:`Scrapy commands that need a CrawlerProcess
|
If ``False``, :ref:`Scrapy commands that need a CrawlerProcess
|
||||||
<topics-commands-crawlerprocess>`, when :setting:`TWISTED_REACTOR_ENABLED`
|
<topics-commands-crawlerprocess>` will decide between using
|
||||||
is set to ``True``, will decide between using
|
|
||||||
:class:`scrapy.crawler.AsyncCrawlerProcess` and
|
:class:`scrapy.crawler.AsyncCrawlerProcess` and
|
||||||
:class:`scrapy.crawler.CrawlerProcess` based on the value of the
|
:class:`scrapy.crawler.CrawlerProcess` based on the value of the
|
||||||
:setting:`TWISTED_REACTOR` setting, but ignoring its value in :ref:`per-spider
|
:setting:`TWISTED_REACTOR` setting, but ignoring its value in :ref:`per-spider
|
||||||
settings <spider-settings>`.
|
settings <spider-settings>`.
|
||||||
|
|
||||||
If ``True``, these commands will always use
|
If ``True``, these commands will always use
|
||||||
:class:`~scrapy.crawler.CrawlerProcess` when :setting:`TWISTED_REACTOR_ENABLED`
|
:class:`~scrapy.crawler.CrawlerProcess`.
|
||||||
is set to ``True``.
|
|
||||||
|
|
||||||
When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``,
|
|
||||||
:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used in all cases.
|
|
||||||
|
|
||||||
Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a
|
Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a
|
||||||
non-default value in :ref:`per-spider settings <spider-settings>`.
|
non-default value in :ref:`per-spider settings <spider-settings>`.
|
||||||
|
|
||||||
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
|
|
||||||
|
|
||||||
.. setting:: FTP_PASSIVE_MODE
|
.. setting:: FTP_PASSIVE_MODE
|
||||||
|
|
||||||
FTP_PASSIVE_MODE
|
FTP_PASSIVE_MODE
|
||||||
|
|
@ -1663,8 +1629,6 @@ The following special items are also supported:
|
||||||
|
|
||||||
- ``Python``
|
- ``Python``
|
||||||
|
|
||||||
- ``pyOpenSSL``
|
|
||||||
|
|
||||||
.. setting:: LOGSTATS_INTERVAL
|
.. setting:: LOGSTATS_INTERVAL
|
||||||
|
|
||||||
LOGSTATS_INTERVAL
|
LOGSTATS_INTERVAL
|
||||||
|
|
@ -1684,6 +1648,21 @@ Default: ``False``
|
||||||
|
|
||||||
Whether to enable memory debugging.
|
Whether to enable memory debugging.
|
||||||
|
|
||||||
|
.. setting:: MEMDEBUG_NOTIFY
|
||||||
|
|
||||||
|
MEMDEBUG_NOTIFY
|
||||||
|
---------------
|
||||||
|
|
||||||
|
Default: ``[]``
|
||||||
|
|
||||||
|
When memory debugging is enabled a memory report will be sent to the specified
|
||||||
|
addresses if this setting is not empty, otherwise the report will be written to
|
||||||
|
the log.
|
||||||
|
|
||||||
|
Example::
|
||||||
|
|
||||||
|
MEMDEBUG_NOTIFY = ['user@example.com']
|
||||||
|
|
||||||
.. setting:: MEMUSAGE_ENABLED
|
.. setting:: MEMUSAGE_ENABLED
|
||||||
|
|
||||||
MEMUSAGE_ENABLED
|
MEMUSAGE_ENABLED
|
||||||
|
|
@ -1757,11 +1736,9 @@ Default: ``"<project name>.spiders"`` (:ref:`fallback <default-settings>`: ``""`
|
||||||
|
|
||||||
Module where to create new spiders using the :command:`genspider` command.
|
Module where to create new spiders using the :command:`genspider` command.
|
||||||
|
|
||||||
Example:
|
Example::
|
||||||
|
|
||||||
.. code-block:: python
|
NEWSPIDER_MODULE = 'mybot.spiders_dev'
|
||||||
|
|
||||||
NEWSPIDER_MODULE = "mybot.spiders_dev"
|
|
||||||
|
|
||||||
.. setting:: RANDOMIZE_DOWNLOAD_DELAY
|
.. setting:: RANDOMIZE_DOWNLOAD_DELAY
|
||||||
|
|
||||||
|
|
@ -1779,10 +1756,7 @@ significant similarities in the time between their requests.
|
||||||
|
|
||||||
The randomization policy is the same used by `wget`_ ``--random-wait`` option.
|
The randomization policy is the same used by `wget`_ ``--random-wait`` option.
|
||||||
|
|
||||||
If :setting:`DOWNLOAD_DELAY` is zero this option has no effect.
|
If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect.
|
||||||
|
|
||||||
It is possible to change this setting per domain by using
|
|
||||||
:setting:`DOWNLOAD_SLOTS`.
|
|
||||||
|
|
||||||
.. _wget: https://www.gnu.org/software/wget/manual/wget.html
|
.. _wget: https://www.gnu.org/software/wget/manual/wget.html
|
||||||
|
|
||||||
|
|
@ -1843,7 +1817,7 @@ The parser backend to use for parsing ``robots.txt`` files. For more information
|
||||||
.. setting:: ROBOTSTXT_USER_AGENT
|
.. setting:: ROBOTSTXT_USER_AGENT
|
||||||
|
|
||||||
ROBOTSTXT_USER_AGENT
|
ROBOTSTXT_USER_AGENT
|
||||||
--------------------
|
^^^^^^^^^^^^^^^^^^^^
|
||||||
|
|
||||||
Default: ``None``
|
Default: ``None``
|
||||||
|
|
||||||
|
|
@ -1871,8 +1845,7 @@ Default: ``False``
|
||||||
|
|
||||||
Setting to ``True`` will log debug information about the requests scheduler.
|
Setting to ``True`` will log debug information about the requests scheduler.
|
||||||
This currently logs (only once) if the requests cannot be serialized to disk.
|
This currently logs (only once) if the requests cannot be serialized to disk.
|
||||||
The :stat:`scheduler/unserializable` stat tracks the number of times this
|
Stats counter (``scheduler/unserializable``) tracks the number of times this happens.
|
||||||
happens.
|
|
||||||
|
|
||||||
Example entry in logs::
|
Example entry in logs::
|
||||||
|
|
||||||
|
|
@ -1999,8 +1972,6 @@ Default:
|
||||||
|
|
||||||
{
|
{
|
||||||
"scrapy.contracts.default.UrlContract": 1,
|
"scrapy.contracts.default.UrlContract": 1,
|
||||||
"scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
|
|
||||||
"scrapy.contracts.default.MetadataContract": 1,
|
|
||||||
"scrapy.contracts.default.ReturnsContract": 2,
|
"scrapy.contracts.default.ReturnsContract": 2,
|
||||||
"scrapy.contracts.default.ScrapesContract": 3,
|
"scrapy.contracts.default.ScrapesContract": 3,
|
||||||
}
|
}
|
||||||
|
|
@ -2065,7 +2036,6 @@ Default:
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
{
|
{
|
||||||
"scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25,
|
|
||||||
"scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
|
"scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
|
||||||
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
|
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
|
||||||
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
|
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
|
||||||
|
|
@ -2141,25 +2111,6 @@ command.
|
||||||
The project name must not conflict with the name of custom files or directories
|
The project name must not conflict with the name of custom files or directories
|
||||||
in the ``project`` subdirectory.
|
in the ``project`` subdirectory.
|
||||||
|
|
||||||
.. setting:: TWISTED_DNS_RESOLVER
|
|
||||||
|
|
||||||
TWISTED_DNS_RESOLVER
|
|
||||||
--------------------
|
|
||||||
|
|
||||||
Default: ``'scrapy.resolver.CachingThreadedResolver'``
|
|
||||||
|
|
||||||
The class to be used by Twisted to resolve DNS names. The default
|
|
||||||
``scrapy.resolver.CachingThreadedResolver`` supports specifying a timeout for
|
|
||||||
DNS requests via the :setting:`DNS_TIMEOUT` setting, but works only with IPv4
|
|
||||||
addresses. Scrapy provides an alternative resolver,
|
|
||||||
``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
|
|
||||||
take the :setting:`DNS_TIMEOUT` setting into account.
|
|
||||||
|
|
||||||
.. note::
|
|
||||||
This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
|
||||||
|
|
||||||
.. note:: This is a :ref:`reactor setting <reactor-settings>`.
|
|
||||||
|
|
||||||
.. setting:: TWISTED_REACTOR_ENABLED
|
.. setting:: TWISTED_REACTOR_ENABLED
|
||||||
|
|
||||||
TWISTED_REACTOR_ENABLED
|
TWISTED_REACTOR_ENABLED
|
||||||
|
|
@ -2198,9 +2149,6 @@ Default: ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``
|
||||||
|
|
||||||
Import path of a given :mod:`~twisted.internet.reactor`.
|
Import path of a given :mod:`~twisted.internet.reactor`.
|
||||||
|
|
||||||
.. note::
|
|
||||||
This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
|
|
||||||
|
|
||||||
Scrapy will install this reactor if no other reactor is installed yet, such as
|
Scrapy will install this reactor if no other reactor is installed yet, such as
|
||||||
when the ``scrapy`` CLI program is invoked or when using the
|
when the ``scrapy`` CLI program is invoked or when using the
|
||||||
:class:`~scrapy.crawler.AsyncCrawlerProcess` class or the
|
:class:`~scrapy.crawler.AsyncCrawlerProcess` class or the
|
||||||
|
|
@ -2237,7 +2185,7 @@ In order to use the reactor installed by Scrapy:
|
||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
self.timeout = int(kwargs.pop("timeout", "60"))
|
self.timeout = int(kwargs.pop("timeout", "60"))
|
||||||
super().__init__(*args, **kwargs)
|
super(QuotesSpider, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
async def start(self):
|
async def start(self):
|
||||||
reactor.callLater(self.timeout, self.stop)
|
reactor.callLater(self.timeout, self.stop)
|
||||||
|
|
@ -2266,7 +2214,7 @@ which raises an exception, becomes:
|
||||||
|
|
||||||
def __init__(self, *args, **kwargs):
|
def __init__(self, *args, **kwargs):
|
||||||
self.timeout = int(kwargs.pop("timeout", "60"))
|
self.timeout = int(kwargs.pop("timeout", "60"))
|
||||||
super().__init__(*args, **kwargs)
|
super(QuotesSpider, self).__init__(*args, **kwargs)
|
||||||
|
|
||||||
async def start(self):
|
async def start(self):
|
||||||
from twisted.internet import reactor
|
from twisted.internet import reactor
|
||||||
|
|
@ -2304,7 +2252,7 @@ URLLENGTH_LIMIT
|
||||||
|
|
||||||
Default: ``2083``
|
Default: ``2083``
|
||||||
|
|
||||||
Scope: ``scrapy.spidermiddlewares.urllength``
|
Scope: ``spidermiddlewares.urllength``
|
||||||
|
|
||||||
The maximum URL length to allow for crawled URLs.
|
The maximum URL length to allow for crawled URLs.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -17,35 +17,30 @@ spider, without having to run the spider to test every change.
|
||||||
Once you get familiarized with the Scrapy shell, you'll see that it's an
|
Once you get familiarized with the Scrapy shell, you'll see that it's an
|
||||||
invaluable tool for developing and debugging your spiders.
|
invaluable tool for developing and debugging your spiders.
|
||||||
|
|
||||||
.. _shell-config:
|
|
||||||
|
|
||||||
Configuring the shell
|
Configuring the shell
|
||||||
=====================
|
=====================
|
||||||
|
|
||||||
With the :ref:`ptpython <extras>` extra, the Scrapy shell will use ptpython_
|
If you have `IPython`_ installed, the Scrapy shell will use it (instead of the
|
||||||
instead of the :term:`REPL`. ptpython provides syntax highlighting, smart
|
standard Python console). The `IPython`_ console is much more powerful and
|
||||||
auto-completion, and more.
|
provides smart auto-completion and colorized output, among other things.
|
||||||
|
|
||||||
Failing that, with the :ref:`ipython <extras>` extra, the Scrapy shell will
|
We highly recommend you install `IPython`_, especially if you're working on
|
||||||
use IPython_ instead. IPython provides smart auto-completion, colorized
|
Unix systems (where `IPython`_ excels). See the `IPython installation guide`_
|
||||||
output, and more.
|
for more info.
|
||||||
|
|
||||||
Scrapy also has support for `bpython`_ via the :ref:`bpython <extras>` extra,
|
Scrapy also has support for `bpython`_, and will try to use it where `IPython`_
|
||||||
and will try to use it where neither ptpython nor IPython is available.
|
is unavailable.
|
||||||
|
|
||||||
Through Scrapy's settings you can configure it to use any one of
|
Through Scrapy's settings you can configure it to use any one of
|
||||||
``ptpython``, ``ipython``, ``bpython`` or the standard ``python`` shell,
|
``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
|
||||||
regardless of which are installed. This is done by setting the
|
are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
|
||||||
``SCRAPY_PYTHON_SHELL`` environment variable; or by defining it in your
|
variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
|
||||||
:ref:`scrapy.cfg <topics-config-settings>`:
|
|
||||||
|
|
||||||
.. code-block:: ini
|
|
||||||
|
|
||||||
[settings]
|
[settings]
|
||||||
shell = bpython
|
shell = bpython
|
||||||
|
|
||||||
.. _ptpython: https://github.com/prompt-toolkit/ptpython
|
|
||||||
.. _IPython: https://ipython.org/
|
.. _IPython: https://ipython.org/
|
||||||
|
.. _IPython installation guide: https://ipython.org/install/
|
||||||
.. _bpython: https://bpython-interpreter.org/
|
.. _bpython: https://bpython-interpreter.org/
|
||||||
|
|
||||||
Launch the shell
|
Launch the shell
|
||||||
|
|
|
||||||
|
|
@ -34,7 +34,7 @@ Here is a simple example showing how you can catch signals and perform some acti
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_crawler(cls, crawler, *args, **kwargs):
|
def from_crawler(cls, crawler, *args, **kwargs):
|
||||||
spider = super().from_crawler(crawler, *args, **kwargs)
|
spider = super(DmozSpider, cls).from_crawler(crawler, *args, **kwargs)
|
||||||
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
|
crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
|
||||||
return spider
|
return spider
|
||||||
|
|
||||||
|
|
@ -60,8 +60,6 @@ Let's take an example using :ref:`coroutines <topics-coroutines>`:
|
||||||
.. skip: next
|
.. skip: next
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
import json
|
|
||||||
|
|
||||||
import scrapy
|
import scrapy
|
||||||
import treq
|
import treq
|
||||||
|
|
||||||
|
|
@ -72,7 +70,7 @@ Let's take an example using :ref:`coroutines <topics-coroutines>`:
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_crawler(cls, crawler, *args, **kwargs):
|
def from_crawler(cls, crawler, *args, **kwargs):
|
||||||
spider = super().from_crawler(crawler, *args, **kwargs)
|
spider = super(SignalSpider, cls).from_crawler(crawler, *args, **kwargs)
|
||||||
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
|
crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
|
||||||
return spider
|
return spider
|
||||||
|
|
||||||
|
|
@ -454,7 +452,7 @@ bytes_received
|
||||||
.. signal:: bytes_received
|
.. signal:: bytes_received
|
||||||
.. function:: bytes_received(data, request, spider)
|
.. function:: bytes_received(data, request, spider)
|
||||||
|
|
||||||
Sent by some download handlers when a group of bytes is
|
Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
|
||||||
received for a specific request. This signal might be fired multiple
|
received for a specific request. This signal might be fired multiple
|
||||||
times for the same request, with partial data each time. For instance,
|
times for the same request, with partial data each time. For instance,
|
||||||
a possible scenario for a 25 kb response would be two signals fired
|
a possible scenario for a 25 kb response would be two signals fired
|
||||||
|
|
@ -482,7 +480,7 @@ headers_received
|
||||||
.. signal:: headers_received
|
.. signal:: headers_received
|
||||||
.. function:: headers_received(headers, body_length, request, spider)
|
.. function:: headers_received(headers, body_length, request, spider)
|
||||||
|
|
||||||
Sent by some download handlers when the response headers are
|
Sent by the HTTP 1.1 and S3 download handlers when the response headers are
|
||||||
available for a given request, before downloading any additional content.
|
available for a given request, before downloading any additional content.
|
||||||
|
|
||||||
Handlers for this signal can stop the download of a response while it
|
Handlers for this signal can stop the download of a response while it
|
||||||
|
|
@ -504,27 +502,6 @@ headers_received
|
||||||
:param spider: the spider associated with the response
|
:param spider: the spider associated with the response
|
||||||
:type spider: :class:`~scrapy.Spider` object
|
:type spider: :class:`~scrapy.Spider` object
|
||||||
|
|
||||||
robots_parsed
|
|
||||||
~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
.. signal:: robots_parsed
|
|
||||||
.. function:: robots_parsed(robotparser, request)
|
|
||||||
|
|
||||||
.. versionadded:: VERSION
|
|
||||||
|
|
||||||
Sent by
|
|
||||||
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it
|
|
||||||
downloads and parses a :file:`robots.txt` file, for the host that *request*
|
|
||||||
targets.
|
|
||||||
|
|
||||||
This signal supports :ref:`asynchronous handlers <signal-deferred>`.
|
|
||||||
|
|
||||||
:param robotparser: the parser holding the parsed :file:`robots.txt` contents
|
|
||||||
:type robotparser: :class:`~scrapy.robotstxt.RobotParser` object
|
|
||||||
|
|
||||||
:param request: the request that triggered the :file:`robots.txt` download
|
|
||||||
:type request: :class:`~scrapy.Request` object
|
|
||||||
|
|
||||||
|
|
||||||
Response signals
|
Response signals
|
||||||
----------------
|
----------------
|
||||||
|
|
|
||||||
|
|
@ -46,7 +46,7 @@ previous (or subsequent) middleware being applied.
|
||||||
If you want to disable a builtin middleware (the ones defined in
|
If you want to disable a builtin middleware (the ones defined in
|
||||||
:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
|
:setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
|
||||||
in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
|
in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
|
||||||
value. For example, if you want to disable the referer middleware:
|
value. For example, if you want to disable the off-site middleware:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
|
|
@ -316,35 +316,6 @@ Default: ``False``
|
||||||
Pass all responses, regardless of its status code.
|
Pass all responses, regardless of its status code.
|
||||||
|
|
||||||
|
|
||||||
MetaCopyDetectionMiddleware
|
|
||||||
---------------------------
|
|
||||||
|
|
||||||
.. module:: scrapy.spidermiddlewares.metacopy
|
|
||||||
:synopsis: Meta Copy Detection Spider Middleware
|
|
||||||
|
|
||||||
.. class:: MetaCopyDetectionMiddleware
|
|
||||||
|
|
||||||
Warns when a spider yields a request that contains internal meta keys which
|
|
||||||
should not be copied from :attr:`response.meta <scrapy.http.Response.meta>`
|
|
||||||
into new requests. See :attr:`~scrapy.http.Request.meta` to learn why.
|
|
||||||
|
|
||||||
Only 1 warning is emitted per crawl.
|
|
||||||
|
|
||||||
MetaCopyDetectionMiddleware settings
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
.. setting:: META_COPY_WARN_SKIP_KEYS
|
|
||||||
|
|
||||||
META_COPY_WARN_SKIP_KEYS
|
|
||||||
^^^^^^^^^^^^^^^^^^^^^^^^
|
|
||||||
|
|
||||||
Default: ``[]``
|
|
||||||
|
|
||||||
A list of internal meta key names to exclude from the internal-keys check.
|
|
||||||
Use this when you intentionally copy one of the monitored keys and want to
|
|
||||||
suppress the resulting warning without disabling the middleware entirely.
|
|
||||||
|
|
||||||
|
|
||||||
RefererMiddleware
|
RefererMiddleware
|
||||||
-----------------
|
-----------------
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,31 +4,43 @@
|
||||||
Spiders
|
Spiders
|
||||||
=======
|
=======
|
||||||
|
|
||||||
Spiders are classes that define how a site, or a group of sites, is scraped:
|
Spiders are classes which define how a certain site (or a group of sites) will be
|
||||||
which requests to send, and how to parse their responses to extract data and to
|
scraped, including how to perform the crawl (i.e. follow links) and how to
|
||||||
send additional requests.
|
extract structured data from their pages (i.e. scraping items). In other words,
|
||||||
|
Spiders are the place where you define the custom behaviour for crawling and
|
||||||
|
parsing pages for a particular site (or, in some cases, a group of sites).
|
||||||
|
|
||||||
A crawl goes as follows:
|
For spiders, the scraping cycle goes through something like this:
|
||||||
|
|
||||||
1. Scrapy iterates the :meth:`~scrapy.Spider.start` method of the spider to
|
1. You start by generating the initial requests to crawl the first URLs, and
|
||||||
get the initial requests. By default, that method yields a
|
specify a callback function to be called with the response downloaded from
|
||||||
:class:`~scrapy.Request` object for each URL in
|
those requests.
|
||||||
:attr:`~scrapy.Spider.start_urls`, with :meth:`~scrapy.Spider.parse` as
|
|
||||||
:ref:`callback <callbacks>`.
|
|
||||||
|
|
||||||
2. Scrapy downloads each request and calls its callback with the resulting
|
The first requests to perform are obtained by iterating the
|
||||||
:class:`~scrapy.http.Response`.
|
:meth:`~scrapy.Spider.start` method, which by default yields a
|
||||||
|
:class:`~scrapy.Request` object for each URL in the
|
||||||
|
:attr:`~scrapy.Spider.start_urls` spider attribute, with the
|
||||||
|
:attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback`
|
||||||
|
function to handle each :class:`~scrapy.http.Response`.
|
||||||
|
|
||||||
3. Callbacks parse the response, typically using :ref:`topics-selectors`, and
|
2. In the callback function, you parse the response (web page) and return
|
||||||
return or yield :ref:`item objects <topics-items>` with the extracted data
|
:ref:`item objects <topics-items>`,
|
||||||
and :class:`~scrapy.Request` objects to continue the crawl, which go back
|
:class:`~scrapy.Request` objects, or an iterable of these objects.
|
||||||
to step 2. See :ref:`callback-output`.
|
Those Requests will also contain a callback (maybe
|
||||||
|
the same) and will then be downloaded by Scrapy and then their
|
||||||
|
response handled by the specified callback.
|
||||||
|
|
||||||
4. Items go through :ref:`item pipelines <topics-item-pipeline>`, and are
|
3. In callback functions, you parse the page contents, typically using
|
||||||
usually stored through :ref:`topics-feed-exports`.
|
:ref:`topics-selectors` (but you can also use BeautifulSoup, lxml or whatever
|
||||||
|
mechanism you prefer) and generate items with the parsed data.
|
||||||
|
|
||||||
Scrapy includes different spider classes for different purposes, described
|
4. Finally, the items returned from the spider will be typically persisted to a
|
||||||
below.
|
database (in some :ref:`Item Pipeline <topics-item-pipeline>`) or written to
|
||||||
|
a file using :ref:`topics-feed-exports`.
|
||||||
|
|
||||||
|
Even though this cycle applies (more or less) to any kind of spider, there are
|
||||||
|
different kinds of default spiders bundled into Scrapy for different purposes.
|
||||||
|
We will talk about those types here.
|
||||||
|
|
||||||
.. _topics-spiders-ref:
|
.. _topics-spiders-ref:
|
||||||
|
|
||||||
|
|
@ -179,7 +191,28 @@ scrapy.Spider
|
||||||
|
|
||||||
.. automethod:: start
|
.. automethod:: start
|
||||||
|
|
||||||
.. automethod:: parse
|
.. method:: parse(response)
|
||||||
|
|
||||||
|
This is the default callback used by Scrapy to process downloaded
|
||||||
|
responses, when their requests don't specify a callback.
|
||||||
|
|
||||||
|
The ``parse`` method is in charge of processing the response and returning
|
||||||
|
scraped data and/or more URLs to follow. Other Requests callbacks have
|
||||||
|
the same requirements as the :class:`~scrapy.Spider` class.
|
||||||
|
|
||||||
|
This method, as well as any other Request callback, must return a
|
||||||
|
:class:`~scrapy.Request` object, an :ref:`item object <topics-items>`, an
|
||||||
|
iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects
|
||||||
|
<topics-items>`, or ``None``.
|
||||||
|
|
||||||
|
:param response: the response to parse
|
||||||
|
:type response: :class:`~scrapy.http.Response`
|
||||||
|
|
||||||
|
.. method:: log(message, [level, component])
|
||||||
|
|
||||||
|
Wrapper that sends a log message through the Spider's :attr:`logger`,
|
||||||
|
kept for backward compatibility. For more information see
|
||||||
|
:ref:`topics-logging-from-spiders`.
|
||||||
|
|
||||||
.. method:: closed(reason)
|
.. method:: closed(reason)
|
||||||
|
|
||||||
|
|
@ -281,7 +314,7 @@ Spiders can access arguments in their `__init__` methods:
|
||||||
name = "myspider"
|
name = "myspider"
|
||||||
|
|
||||||
def __init__(self, category=None, *args, **kwargs):
|
def __init__(self, category=None, *args, **kwargs):
|
||||||
super().__init__(*args, **kwargs)
|
super(MySpider, self).__init__(*args, **kwargs)
|
||||||
self.start_urls = [f"http://www.example.com/categories/{category}"]
|
self.start_urls = [f"http://www.example.com/categories/{category}"]
|
||||||
# ...
|
# ...
|
||||||
|
|
||||||
|
|
@ -302,8 +335,8 @@ The above example can also be written as follows:
|
||||||
|
|
||||||
If you are :ref:`running Scrapy from a script <run-from-script>`, you can
|
If you are :ref:`running Scrapy from a script <run-from-script>`, you can
|
||||||
specify spider arguments when calling
|
specify spider arguments when calling
|
||||||
:meth:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
|
:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
|
||||||
:meth:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`:
|
:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`:
|
||||||
|
|
||||||
.. skip: next
|
.. skip: next
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
@ -560,7 +593,7 @@ Let's now take a look at an example CrawlSpider with rules:
|
||||||
This spider would start crawling example.com's home page, collecting category
|
This spider would start crawling example.com's home page, collecting category
|
||||||
links, and item links, parsing the latter with the ``parse_item`` method. For
|
links, and item links, parsing the latter with the ``parse_item`` method. For
|
||||||
each item response, some data will be extracted from the HTML using XPath, and
|
each item response, some data will be extracted from the HTML using XPath, and
|
||||||
a dictionary will be filled with it.
|
an :class:`~scrapy.Item` will be filled with it.
|
||||||
|
|
||||||
XMLFeedSpider
|
XMLFeedSpider
|
||||||
-------------
|
-------------
|
||||||
|
|
@ -581,7 +614,7 @@ XMLFeedSpider
|
||||||
|
|
||||||
A string which defines the iterator to use. It can be either:
|
A string which defines the iterator to use. It can be either:
|
||||||
|
|
||||||
- ``'iternodes'`` - a fast iterator based on ``lxml``
|
- ``'iternodes'`` - a fast iterator based on regular expressions
|
||||||
|
|
||||||
- ``'html'`` - an iterator which uses :class:`~scrapy.Selector`.
|
- ``'html'`` - an iterator which uses :class:`~scrapy.Selector`.
|
||||||
Keep in mind this uses DOM parsing and must load all DOM in memory
|
Keep in mind this uses DOM parsing and must load all DOM in memory
|
||||||
|
|
@ -595,11 +628,9 @@ XMLFeedSpider
|
||||||
|
|
||||||
.. attribute:: itertag
|
.. attribute:: itertag
|
||||||
|
|
||||||
A string with the name of the node (or element) to iterate in. Example:
|
A string with the name of the node (or element) to iterate in. Example::
|
||||||
|
|
||||||
.. code-block:: python
|
itertag = 'product'
|
||||||
|
|
||||||
itertag = "product"
|
|
||||||
|
|
||||||
.. attribute:: namespaces
|
.. attribute:: namespaces
|
||||||
|
|
||||||
|
|
@ -612,17 +643,12 @@ XMLFeedSpider
|
||||||
You can then specify nodes with namespaces in the :attr:`itertag`
|
You can then specify nodes with namespaces in the :attr:`itertag`
|
||||||
attribute.
|
attribute.
|
||||||
|
|
||||||
Example:
|
Example::
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
from scrapy.spiders import XMLFeedSpider
|
|
||||||
|
|
||||||
|
|
||||||
class YourSpider(XMLFeedSpider):
|
class YourSpider(XMLFeedSpider):
|
||||||
|
|
||||||
namespaces = [("n", "http://www.sitemaps.org/schemas/sitemap/0.9")]
|
namespaces = [('n', 'http://www.sitemaps.org/schemas/sitemap/0.9')]
|
||||||
itertag = "n:url"
|
itertag = 'n:url'
|
||||||
# ...
|
# ...
|
||||||
|
|
||||||
Apart from these new attributes, this spider has the following overridable
|
Apart from these new attributes, this spider has the following overridable
|
||||||
|
|
@ -782,11 +808,9 @@ SitemapSpider
|
||||||
the regular expression. ``callback`` can be a string (indicating the
|
the regular expression. ``callback`` can be a string (indicating the
|
||||||
name of a spider method) or a callable.
|
name of a spider method) or a callable.
|
||||||
|
|
||||||
For example:
|
For example::
|
||||||
|
|
||||||
.. code-block:: python
|
sitemap_rules = [('/product/', 'parse_product')]
|
||||||
|
|
||||||
sitemap_rules = [("/product/", "parse_product")]
|
|
||||||
|
|
||||||
Rules are applied in order, and only the first one that matches will be
|
Rules are applied in order, and only the first one that matches will be
|
||||||
used.
|
used.
|
||||||
|
|
@ -808,9 +832,7 @@ SitemapSpider
|
||||||
are links for the same website in another language passed within
|
are links for the same website in another language passed within
|
||||||
the same ``url`` block.
|
the same ``url`` block.
|
||||||
|
|
||||||
For example:
|
For example::
|
||||||
|
|
||||||
.. code-block:: xml
|
|
||||||
|
|
||||||
<url>
|
<url>
|
||||||
<loc>http://example.com/</loc>
|
<loc>http://example.com/</loc>
|
||||||
|
|
@ -828,9 +850,7 @@ SitemapSpider
|
||||||
This is a filter function that could be overridden to select sitemap entries
|
This is a filter function that could be overridden to select sitemap entries
|
||||||
based on their attributes.
|
based on their attributes.
|
||||||
|
|
||||||
For example:
|
For example::
|
||||||
|
|
||||||
.. code-block:: xml
|
|
||||||
|
|
||||||
<url>
|
<url>
|
||||||
<loc>http://example.com/</loc>
|
<loc>http://example.com/</loc>
|
||||||
|
|
@ -933,7 +953,6 @@ Combine SitemapSpider with other sources of urls:
|
||||||
|
|
||||||
.. code-block:: python
|
.. code-block:: python
|
||||||
|
|
||||||
from scrapy import Request
|
|
||||||
from scrapy.spiders import SitemapSpider
|
from scrapy.spiders import SitemapSpider
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,8 +10,8 @@ Collector, and can be accessed through the :attr:`~scrapy.crawler.Crawler.stats`
|
||||||
attribute of the :ref:`topics-api-crawler`, as illustrated by the examples in
|
attribute of the :ref:`topics-api-crawler`, as illustrated by the examples in
|
||||||
the :ref:`topics-stats-usecases` section below.
|
the :ref:`topics-stats-usecases` section below.
|
||||||
|
|
||||||
The Stats Collector API is always available, so you can always use it (to
|
However, the Stats Collector is always available, so you can always import it
|
||||||
increment or set new stat keys), regardless
|
in your module and use its API (to increment or set new stat keys), regardless
|
||||||
of whether the stats collection is enabled or not. If it's disabled, the API
|
of whether the stats collection is enabled or not. If it's disabled, the API
|
||||||
will still work but it won't collect anything. This is aimed at simplifying the
|
will still work but it won't collect anything. This is aimed at simplifying the
|
||||||
stats collector usage: you should spend no more than one line of code for
|
stats collector usage: you should spend no more than one line of code for
|
||||||
|
|
@ -21,7 +21,8 @@ using the Stats Collector from.
|
||||||
Another feature of the Stats Collector is that it's very efficient (when
|
Another feature of the Stats Collector is that it's very efficient (when
|
||||||
enabled) and extremely efficient (almost unnoticeable) when disabled.
|
enabled) and extremely efficient (almost unnoticeable) when disabled.
|
||||||
|
|
||||||
See :ref:`topics-stats-reference` below for the stats that Scrapy sets.
|
The Stats Collector keeps a stats table per open spider which is automatically
|
||||||
|
opened when the spider is opened, and closed when the spider is closed.
|
||||||
|
|
||||||
.. _topics-stats-usecases:
|
.. _topics-stats-usecases:
|
||||||
|
|
||||||
|
|
@ -86,659 +87,37 @@ Get all stats:
|
||||||
Available Stats Collectors
|
Available Stats Collectors
|
||||||
==========================
|
==========================
|
||||||
|
|
||||||
.. currentmodule:: scrapy.statscollectors
|
|
||||||
|
|
||||||
Besides the basic :class:`StatsCollector` there are other Stats Collectors
|
Besides the basic :class:`StatsCollector` there are other Stats Collectors
|
||||||
available in Scrapy which extend the basic Stats Collector. You can select
|
available in Scrapy which extend the basic Stats Collector. You can select
|
||||||
which Stats Collector to use through the :setting:`STATS_CLASS` setting. The
|
which Stats Collector to use through the :setting:`STATS_CLASS` setting. The
|
||||||
default Stats Collector used is the :class:`MemoryStatsCollector`.
|
default Stats Collector used is the :class:`MemoryStatsCollector`.
|
||||||
|
|
||||||
|
.. currentmodule:: scrapy.statscollectors
|
||||||
|
|
||||||
MemoryStatsCollector
|
MemoryStatsCollector
|
||||||
--------------------
|
--------------------
|
||||||
|
|
||||||
.. autoclass:: MemoryStatsCollector
|
.. class:: MemoryStatsCollector
|
||||||
:members:
|
|
||||||
|
A simple stats collector that keeps the stats of the last scraping run (for
|
||||||
|
each spider) in memory, after they're closed. The stats can be accessed
|
||||||
|
through the :attr:`spider_stats` attribute, which is a dict keyed by spider
|
||||||
|
domain name.
|
||||||
|
|
||||||
|
This is the default Stats Collector used in Scrapy.
|
||||||
|
|
||||||
|
.. attribute:: spider_stats
|
||||||
|
|
||||||
|
A dict of dicts (keyed by spider name) containing the stats of the last
|
||||||
|
scraping run for each spider.
|
||||||
|
|
||||||
DummyStatsCollector
|
DummyStatsCollector
|
||||||
-------------------
|
-------------------
|
||||||
|
|
||||||
.. autoclass:: DummyStatsCollector
|
.. class:: DummyStatsCollector
|
||||||
|
|
||||||
.. _topics-stats-reference:
|
A Stats collector which does nothing but is very efficient (because it does
|
||||||
|
nothing). This stats collector can be set via the :setting:`STATS_CLASS`
|
||||||
Built-in stats reference
|
setting, to disable stats collect in order to improve performance. However,
|
||||||
========================
|
the performance penalty of stats collection is usually marginal compared to
|
||||||
|
other Scrapy workload like parsing pages.
|
||||||
Scrapy sets the following :ref:`stats <topics-stats>`. Components other than
|
|
||||||
those built into Scrapy may set additional stats; see their documentation.
|
|
||||||
|
|
||||||
Stat keys that contain a ``{placeholder}`` below stand for a family of stats,
|
|
||||||
one per actual value of the placeholder.
|
|
||||||
|
|
||||||
.. note:: Most stats are set by a specific :ref:`component
|
|
||||||
<topics-components>`, and are only present if that component is enabled and
|
|
||||||
its code path is reached. A stat that is missing from
|
|
||||||
:meth:`~scrapy.statscollectors.StatsCollector.get_stats` output is
|
|
||||||
equivalent to a counter of 0.
|
|
||||||
|
|
||||||
.. stat:: downloader/exception_count
|
|
||||||
|
|
||||||
``downloader/exception_count``
|
|
||||||
Number of exceptions raised while downloading requests.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
|
||||||
|
|
||||||
.. stat:: downloader/exception_type_count/{exception_type}
|
|
||||||
|
|
||||||
``downloader/exception_type_count/{exception_type}``
|
|
||||||
Number of exceptions raised while downloading requests, per exception type,
|
|
||||||
where ``{exception_type}`` is the import path of the exception class, e.g.
|
|
||||||
``twisted.internet.error.DNSLookupError``.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
|
||||||
|
|
||||||
.. stat:: downloader/request_bytes
|
|
||||||
|
|
||||||
``downloader/request_bytes``
|
|
||||||
Total size, in bytes, of the requests sent, counting the request line, the
|
|
||||||
headers and the body. As with :stat:`downloader/request_count`, requests
|
|
||||||
served from the cache are also counted.
|
|
||||||
|
|
||||||
It is an approximation, reconstructed from each :class:`~scrapy.Request`
|
|
||||||
object instead of measured on the wire, so it does not account for the
|
|
||||||
actual bytes that the :ref:`download handler
|
|
||||||
<topics-download-handlers>` sends, e.g. transport-level overhead.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
|
||||||
|
|
||||||
.. stat:: downloader/request_count
|
|
||||||
|
|
||||||
``downloader/request_count``
|
|
||||||
Number of requests sent.
|
|
||||||
|
|
||||||
Requests that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
|
|
||||||
serves from the cache are also counted, even though they are never sent,
|
|
||||||
because it handles requests after
|
|
||||||
:class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
|
||||||
|
|
||||||
.. stat:: downloader/request_method_count/{method}
|
|
||||||
|
|
||||||
``downloader/request_method_count/{method}``
|
|
||||||
Number of requests sent, per HTTP method, e.g. ``GET`` or ``POST``. As with
|
|
||||||
:stat:`downloader/request_count`, requests served from the cache are also
|
|
||||||
counted.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
|
||||||
|
|
||||||
.. stat:: downloader/response_bytes
|
|
||||||
|
|
||||||
``downloader/response_bytes``
|
|
||||||
Total size, in bytes, of the responses received, counting the status line,
|
|
||||||
the headers and the body. It covers the same responses as
|
|
||||||
:stat:`downloader/response_count`.
|
|
||||||
|
|
||||||
The body is counted as received, i.e. still compressed for responses that
|
|
||||||
used ``Content-Encoding``, because
|
|
||||||
:class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` handles
|
|
||||||
responses before
|
|
||||||
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
|
|
||||||
decompresses them. See :stat:`httpcompression/response_bytes` for
|
|
||||||
decompressed sizes.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
|
||||||
|
|
||||||
.. stat:: downloader/response_count
|
|
||||||
|
|
||||||
``downloader/response_count``
|
|
||||||
Number of responses received.
|
|
||||||
|
|
||||||
It counts responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
|
|
||||||
serves from the cache, even though they do not come from the network, and
|
|
||||||
responses that a downloader middleware consumes before they reach your
|
|
||||||
spider, e.g. redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware`
|
|
||||||
turns into new requests. Compare with :stat:`response_received_count`.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
|
||||||
|
|
||||||
.. stat:: downloader/response_status_count/{status_code}
|
|
||||||
|
|
||||||
``downloader/response_status_count/{status_code}``
|
|
||||||
Number of responses received, per HTTP status code, e.g. ``200`` or
|
|
||||||
``404``. It covers the same responses as :stat:`downloader/response_count`.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
|
||||||
|
|
||||||
.. stat:: dupefilter/filtered
|
|
||||||
|
|
||||||
``dupefilter/filtered``
|
|
||||||
Number of requests dropped as duplicates.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.dupefilters.RFPDupeFilter`.
|
|
||||||
|
|
||||||
.. stat:: elapsed_time_seconds
|
|
||||||
|
|
||||||
``elapsed_time_seconds``
|
|
||||||
Time, as a :class:`float`, in seconds, between the :signal:`spider_opened`
|
|
||||||
and the :signal:`spider_closed` signals.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
|
||||||
|
|
||||||
.. stat:: feedexport/failed_count/{storage}
|
|
||||||
|
|
||||||
``feedexport/failed_count/{storage}``
|
|
||||||
Number of :ref:`feeds <topics-feed-exports>` that could not be stored, per
|
|
||||||
:ref:`storage backend <topics-feed-storage-backends>`, where ``{storage}``
|
|
||||||
is the class name of the storage backend, e.g. ``FileFeedStorage``.
|
|
||||||
|
|
||||||
.. stat:: feedexport/success_count/{storage}
|
|
||||||
|
|
||||||
``feedexport/success_count/{storage}``
|
|
||||||
Number of :ref:`feeds <topics-feed-exports>` stored successfully, per
|
|
||||||
:ref:`storage backend <topics-feed-storage-backends>`, where ``{storage}``
|
|
||||||
is the class name of the storage backend, e.g. ``FileFeedStorage``.
|
|
||||||
|
|
||||||
.. stat:: file_count
|
|
||||||
|
|
||||||
``file_count``
|
|
||||||
Number of files handled by the :ref:`media pipelines
|
|
||||||
<topics-media-pipeline>`.
|
|
||||||
|
|
||||||
.. stat:: file_status_count/{status}
|
|
||||||
|
|
||||||
``file_status_count/{status}``
|
|
||||||
Number of files handled by the :ref:`media pipelines
|
|
||||||
<topics-media-pipeline>`, per status, where ``{status}`` is one of:
|
|
||||||
|
|
||||||
- ``downloaded``: the file was downloaded.
|
|
||||||
|
|
||||||
- ``cached``: the file came from the
|
|
||||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
|
|
||||||
cache.
|
|
||||||
|
|
||||||
- ``uptodate``: the file was already in the storage backend and had not
|
|
||||||
:ref:`expired <file-expiration>`, so it was not downloaded again.
|
|
||||||
|
|
||||||
.. stat:: finish_reason
|
|
||||||
|
|
||||||
``finish_reason``
|
|
||||||
String indicating why the crawl finished. It matches the *reason* argument
|
|
||||||
of the :signal:`spider_closed` signal.
|
|
||||||
|
|
||||||
Scrapy uses the following reasons:
|
|
||||||
|
|
||||||
- ``cancelled``: the spider was closed without a more specific reason,
|
|
||||||
e.g. because :exc:`~scrapy.exceptions.CloseSpider` was raised without
|
|
||||||
one.
|
|
||||||
|
|
||||||
- ``closespider_errorcount``: see :setting:`CLOSESPIDER_ERRORCOUNT`.
|
|
||||||
|
|
||||||
- ``closespider_itemcount``: see :setting:`CLOSESPIDER_ITEMCOUNT`.
|
|
||||||
|
|
||||||
- ``closespider_pagecount``: see :setting:`CLOSESPIDER_PAGECOUNT`.
|
|
||||||
|
|
||||||
- ``closespider_pagecount_no_item``: see
|
|
||||||
:setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM`.
|
|
||||||
|
|
||||||
- ``closespider_timeout``: see :setting:`CLOSESPIDER_TIMEOUT`.
|
|
||||||
|
|
||||||
- ``closespider_timeout_no_item``: see
|
|
||||||
:setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`.
|
|
||||||
|
|
||||||
- ``finished``: the spider became idle with no pending requests, i.e. it
|
|
||||||
finished normally.
|
|
||||||
|
|
||||||
- ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`.
|
|
||||||
|
|
||||||
- ``shutdown``: the crawl was interrupted, e.g. by a system signal such
|
|
||||||
as ``SIGINT`` (:kbd:`Ctrl-C`).
|
|
||||||
|
|
||||||
Third-party components and your own code may use any other reason, e.g. by
|
|
||||||
raising :exc:`~scrapy.exceptions.CloseSpider` with it.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
|
||||||
|
|
||||||
.. stat:: finish_time
|
|
||||||
|
|
||||||
``finish_time``
|
|
||||||
Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when
|
|
||||||
the :signal:`spider_closed` signal was sent.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
|
||||||
|
|
||||||
.. stat:: httpcache/errorrecovery
|
|
||||||
|
|
||||||
``httpcache/errorrecovery``
|
|
||||||
Number of times that a stale cached response was used because downloading a
|
|
||||||
fresh response raised an exception.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: httpcache/firsthand
|
|
||||||
|
|
||||||
``httpcache/firsthand``
|
|
||||||
Number of responses that were downloaded without a matching cache entry to
|
|
||||||
validate against, i.e. responses for requests counted in
|
|
||||||
:stat:`httpcache/miss`.
|
|
||||||
|
|
||||||
It is lower than :stat:`httpcache/miss` when some of those requests yield
|
|
||||||
no response, either because they are dropped (see
|
|
||||||
:stat:`httpcache/ignore`) or because their download fails.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: httpcache/hit
|
|
||||||
|
|
||||||
``httpcache/hit``
|
|
||||||
Number of requests served from the cache.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: httpcache/ignore
|
|
||||||
|
|
||||||
``httpcache/ignore``
|
|
||||||
Number of requests dropped because they were not in the cache and
|
|
||||||
:setting:`HTTPCACHE_IGNORE_MISSING` is ``True``.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: httpcache/invalidate
|
|
||||||
|
|
||||||
``httpcache/invalidate``
|
|
||||||
Number of times that a cached response failed validation and was replaced
|
|
||||||
with a freshly downloaded response.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: httpcache/miss
|
|
||||||
|
|
||||||
``httpcache/miss``
|
|
||||||
Number of requests for which no cache entry could be read, either because
|
|
||||||
there was none or because reading it failed, in which case the request is
|
|
||||||
also counted in :stat:`httpcache/retrieve_error`. Those requests are
|
|
||||||
downloaded (see :stat:`httpcache/firsthand`), or dropped if
|
|
||||||
:setting:`HTTPCACHE_IGNORE_MISSING` is ``True`` (see
|
|
||||||
:stat:`httpcache/ignore`).
|
|
||||||
|
|
||||||
Requests with a stale cache entry are not counted here; see
|
|
||||||
:stat:`httpcache/revalidate` and :stat:`httpcache/invalidate`.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: httpcache/retrieve_error
|
|
||||||
|
|
||||||
``httpcache/retrieve_error``
|
|
||||||
Number of cache entries that could not be read, and hence were treated as
|
|
||||||
cache misses. Those requests are also counted in :stat:`httpcache/miss`.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: httpcache/revalidate
|
|
||||||
|
|
||||||
``httpcache/revalidate``
|
|
||||||
Number of times that a cached response was successfully validated against
|
|
||||||
the target server, and hence used instead of the fresh response.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: httpcache/store
|
|
||||||
|
|
||||||
``httpcache/store``
|
|
||||||
Number of responses stored in the cache.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: httpcache/uncacheable
|
|
||||||
|
|
||||||
``httpcache/uncacheable``
|
|
||||||
Number of responses not stored in the cache because the
|
|
||||||
:setting:`HTTPCACHE_POLICY` did not allow it.
|
|
||||||
|
|
||||||
Every response considered for caching is counted either here or in
|
|
||||||
:stat:`httpcache/store`, so ``httpcache/store + httpcache/uncacheable``
|
|
||||||
equals ``httpcache/firsthand + httpcache/invalidate``.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: httpcompression/response_bytes
|
|
||||||
|
|
||||||
``httpcompression/response_bytes``
|
|
||||||
Total size, in bytes, of decompressed response bodies, counting only the
|
|
||||||
body and only responses that were actually decompressed. Compare with
|
|
||||||
:stat:`downloader/response_bytes`.
|
|
||||||
|
|
||||||
Set by
|
|
||||||
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: httpcompression/response_count
|
|
||||||
|
|
||||||
``httpcompression/response_count``
|
|
||||||
Number of decompressed responses.
|
|
||||||
|
|
||||||
Set by
|
|
||||||
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: httperror/response_ignored_count
|
|
||||||
|
|
||||||
``httperror/response_ignored_count``
|
|
||||||
Number of responses dropped because of their HTTP status code.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: httperror/response_ignored_status_count/{status_code}
|
|
||||||
|
|
||||||
``httperror/response_ignored_status_count/{status_code}``
|
|
||||||
Number of responses dropped because of their HTTP status code, per HTTP
|
|
||||||
status code, e.g. ``404``.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: item_dropped_count
|
|
||||||
|
|
||||||
``item_dropped_count``
|
|
||||||
Number of items dropped by an :ref:`item pipeline
|
|
||||||
<topics-item-pipeline>`, i.e. number of times that the
|
|
||||||
:signal:`item_dropped` signal was sent.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
|
||||||
|
|
||||||
.. stat:: item_dropped_reasons_count/{exception}
|
|
||||||
|
|
||||||
``item_dropped_reasons_count/{exception}``
|
|
||||||
Number of items dropped, per exception, where ``{exception}`` is the class
|
|
||||||
name of the exception that caused the item to be dropped.
|
|
||||||
|
|
||||||
Only :exc:`~scrapy.exceptions.DropItem` and its subclasses drop items, and
|
|
||||||
each one is counted under its own class name, e.g.
|
|
||||||
``item_dropped_reasons_count/DropItem`` for
|
|
||||||
:exc:`~scrapy.exceptions.DropItem` itself and
|
|
||||||
``item_dropped_reasons_count/MyDropItem`` for a ``MyDropItem`` subclass of
|
|
||||||
it. Any other exception raised by an :ref:`item pipeline
|
|
||||||
<topics-item-pipeline>` triggers the :signal:`item_error` signal instead of
|
|
||||||
:signal:`item_dropped`, and is not counted here or in
|
|
||||||
:stat:`item_dropped_count`.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
|
||||||
|
|
||||||
.. stat:: item_scraped_count
|
|
||||||
|
|
||||||
``item_scraped_count``
|
|
||||||
Number of items that passed all :ref:`item pipelines
|
|
||||||
<topics-item-pipeline>`, i.e. number of times that the
|
|
||||||
:signal:`item_scraped` signal was sent.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
|
||||||
|
|
||||||
.. stat:: items_per_minute
|
|
||||||
|
|
||||||
``items_per_minute``
|
|
||||||
Average number of items scraped per minute during the crawl.
|
|
||||||
|
|
||||||
It is ``None`` if the crawl took less than a minute.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.logstats.LogStats`.
|
|
||||||
|
|
||||||
.. stat:: log_count/{level}
|
|
||||||
|
|
||||||
``log_count/{level}``
|
|
||||||
Number of log messages, per logging level name, e.g. ``INFO`` or
|
|
||||||
``WARNING``.
|
|
||||||
|
|
||||||
Only messages that the :setting:`LOG_LEVEL` setting allows are counted.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.logcount.LogCount`.
|
|
||||||
|
|
||||||
.. stat:: memdebug/gc_garbage_count
|
|
||||||
|
|
||||||
``memdebug/gc_garbage_count``
|
|
||||||
Number of objects in :data:`gc.garbage` when the spider is closed.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires
|
|
||||||
:setting:`MEMDEBUG_ENABLED` to be ``True``.
|
|
||||||
|
|
||||||
.. stat:: memdebug/live_refs/{cls}
|
|
||||||
|
|
||||||
``memdebug/live_refs/{cls}``
|
|
||||||
Number of live objects of class ``{cls}`` when the spider is closed, as
|
|
||||||
reported by :ref:`trackref <topics-leaks-trackrefs>`, e.g.
|
|
||||||
``memdebug/live_refs/HtmlResponse``.
|
|
||||||
|
|
||||||
Only set for classes with at least 1 live object.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires
|
|
||||||
:setting:`MEMDEBUG_ENABLED` to be ``True``.
|
|
||||||
|
|
||||||
.. stat:: memusage/limit_reached
|
|
||||||
|
|
||||||
``memusage/limit_reached``
|
|
||||||
``1`` if memory usage exceeded :setting:`MEMUSAGE_LIMIT_MB`, which also
|
|
||||||
stops the crawl.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
|
|
||||||
|
|
||||||
.. stat:: memusage/max
|
|
||||||
|
|
||||||
``memusage/max``
|
|
||||||
Maximum peak memory usage, in bytes, observed during the crawl.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
|
|
||||||
|
|
||||||
.. stat:: memusage/startup
|
|
||||||
|
|
||||||
``memusage/startup``
|
|
||||||
Peak memory usage, in bytes, when the engine started.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
|
|
||||||
|
|
||||||
.. stat:: memusage/warning_reached
|
|
||||||
|
|
||||||
``memusage/warning_reached``
|
|
||||||
``1`` if memory usage exceeded :setting:`MEMUSAGE_WARNING_MB`.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
|
|
||||||
|
|
||||||
.. stat:: offsite/domains
|
|
||||||
|
|
||||||
``offsite/domains``
|
|
||||||
Number of distinct domains for which at least 1 request was dropped for
|
|
||||||
being offsite.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: offsite/filtered
|
|
||||||
|
|
||||||
``offsite/filtered``
|
|
||||||
Number of requests dropped for being offsite.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: request_depth_count/{depth}
|
|
||||||
|
|
||||||
``request_depth_count/{depth}``
|
|
||||||
Number of requests scheduled at depth ``{depth}``, e.g.
|
|
||||||
``request_depth_count/2``.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`, which
|
|
||||||
requires :setting:`DEPTH_STATS_VERBOSE` to be ``True`` for this stat.
|
|
||||||
|
|
||||||
.. stat:: request_depth_max
|
|
||||||
|
|
||||||
``request_depth_max``
|
|
||||||
Maximum depth reached.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: response_received_count
|
|
||||||
|
|
||||||
``response_received_count``
|
|
||||||
Number of responses received, i.e. number of times that the
|
|
||||||
:signal:`response_received` signal was sent.
|
|
||||||
|
|
||||||
Unlike :stat:`downloader/response_count`, it does not count responses that
|
|
||||||
a downloader middleware consumes before they reach the engine, e.g.
|
|
||||||
redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware`
|
|
||||||
turns into new requests. Both count responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
|
|
||||||
serves from the cache.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
|
||||||
|
|
||||||
.. stat:: responses_per_minute
|
|
||||||
|
|
||||||
``responses_per_minute``
|
|
||||||
Average number of responses received per minute during the crawl.
|
|
||||||
|
|
||||||
It is ``None`` if the crawl took less than a minute.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.logstats.LogStats`.
|
|
||||||
|
|
||||||
.. stat:: retry/count
|
|
||||||
|
|
||||||
``retry/count``
|
|
||||||
Number of requests retried.
|
|
||||||
|
|
||||||
Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which
|
|
||||||
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses.
|
|
||||||
|
|
||||||
.. stat:: retry/max_reached
|
|
||||||
|
|
||||||
``retry/max_reached``
|
|
||||||
Number of requests that were not retried because they had already been
|
|
||||||
retried :setting:`RETRY_TIMES` times.
|
|
||||||
|
|
||||||
Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which
|
|
||||||
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses.
|
|
||||||
|
|
||||||
.. stat:: retry/reason_count/{reason}
|
|
||||||
|
|
||||||
``retry/reason_count/{reason}``
|
|
||||||
Number of requests retried, per reason, e.g.
|
|
||||||
``retry/reason_count/twisted.internet.error.TimeoutError`` or
|
|
||||||
``retry/reason_count/504 Gateway Time-out``.
|
|
||||||
|
|
||||||
Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which
|
|
||||||
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses.
|
|
||||||
|
|
||||||
.. note:: Code calling
|
|
||||||
:func:`~scrapy.downloadermiddlewares.retry.get_retry_request` may pass a
|
|
||||||
custom *stats_base_key*, in which case ``retry`` is replaced with that key
|
|
||||||
in the 3 stats above.
|
|
||||||
|
|
||||||
.. stat:: robotstxt/exception_count/{exception_type}
|
|
||||||
|
|
||||||
``robotstxt/exception_count/{exception_type}``
|
|
||||||
Number of exceptions raised while downloading ``robots.txt`` files, per
|
|
||||||
exception type, where ``{exception_type}`` is the string representation of
|
|
||||||
the exception class, e.g. ``<class
|
|
||||||
'twisted.internet.error.DNSLookupError'>``.
|
|
||||||
|
|
||||||
Set by
|
|
||||||
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: robotstxt/forbidden
|
|
||||||
|
|
||||||
``robotstxt/forbidden``
|
|
||||||
Number of requests dropped for being disallowed by ``robots.txt``.
|
|
||||||
|
|
||||||
Set by
|
|
||||||
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: robotstxt/request_count
|
|
||||||
|
|
||||||
``robotstxt/request_count``
|
|
||||||
Number of ``robots.txt`` files requested, i.e. 1 per network location for
|
|
||||||
which at least 1 request was sent.
|
|
||||||
|
|
||||||
Set by
|
|
||||||
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: robotstxt/response_count
|
|
||||||
|
|
||||||
``robotstxt/response_count``
|
|
||||||
Number of ``robots.txt`` responses received.
|
|
||||||
|
|
||||||
Set by
|
|
||||||
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: robotstxt/response_status_count/{status_code}
|
|
||||||
|
|
||||||
``robotstxt/response_status_count/{status_code}``
|
|
||||||
Number of ``robots.txt`` responses received, per HTTP status code, e.g.
|
|
||||||
``404``.
|
|
||||||
|
|
||||||
Set by
|
|
||||||
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
|
|
||||||
|
|
||||||
.. stat:: scheduler/dequeued
|
|
||||||
|
|
||||||
``scheduler/dequeued``
|
|
||||||
Number of requests read from the :ref:`scheduler <topics-scheduler>`.
|
|
||||||
|
|
||||||
.. stat:: scheduler/dequeued/disk
|
|
||||||
|
|
||||||
``scheduler/dequeued/disk``
|
|
||||||
Number of requests read from the disk queue of the :ref:`scheduler
|
|
||||||
<topics-scheduler>`.
|
|
||||||
|
|
||||||
.. stat:: scheduler/dequeued/memory
|
|
||||||
|
|
||||||
``scheduler/dequeued/memory``
|
|
||||||
Number of requests read from the memory queue of the :ref:`scheduler
|
|
||||||
<topics-scheduler>`.
|
|
||||||
|
|
||||||
.. stat:: scheduler/enqueued
|
|
||||||
|
|
||||||
``scheduler/enqueued``
|
|
||||||
Number of requests stored into the :ref:`scheduler <topics-scheduler>`.
|
|
||||||
|
|
||||||
.. stat:: scheduler/enqueued/disk
|
|
||||||
|
|
||||||
``scheduler/enqueued/disk``
|
|
||||||
Number of requests stored into the disk queue of the :ref:`scheduler
|
|
||||||
<topics-scheduler>`.
|
|
||||||
|
|
||||||
.. stat:: scheduler/enqueued/memory
|
|
||||||
|
|
||||||
``scheduler/enqueued/memory``
|
|
||||||
Number of requests stored into the memory queue of the :ref:`scheduler
|
|
||||||
<topics-scheduler>`.
|
|
||||||
|
|
||||||
.. stat:: scheduler/unserializable
|
|
||||||
|
|
||||||
``scheduler/unserializable``
|
|
||||||
Number of requests that could not be stored into the disk queue of the
|
|
||||||
:ref:`scheduler <topics-scheduler>` because they could not be
|
|
||||||
:ref:`serialized <request-serialization>`, and hence were stored into the
|
|
||||||
memory queue instead.
|
|
||||||
|
|
||||||
.. stat:: spider_exceptions/count
|
|
||||||
|
|
||||||
``spider_exceptions/count``
|
|
||||||
Number of unhandled exceptions raised by spider callbacks.
|
|
||||||
|
|
||||||
Set by the :ref:`scraper <topics-architecture>`.
|
|
||||||
|
|
||||||
.. stat:: spider_exceptions/{exception}
|
|
||||||
|
|
||||||
``spider_exceptions/{exception}``
|
|
||||||
Number of unhandled exceptions raised by spider callbacks, per exception,
|
|
||||||
where ``{exception}`` is the class name of the exception, e.g.
|
|
||||||
``spider_exceptions/ValueError``.
|
|
||||||
|
|
||||||
Set by the :ref:`scraper <topics-architecture>`.
|
|
||||||
|
|
||||||
.. stat:: start_time
|
|
||||||
|
|
||||||
``start_time``
|
|
||||||
Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when
|
|
||||||
the :signal:`spider_opened` signal was sent.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
|
||||||
|
|
||||||
.. stat:: urllength/request_ignored_count
|
|
||||||
|
|
||||||
``urllength/request_ignored_count``
|
|
||||||
Number of requests dropped for having a URL longer than
|
|
||||||
:setting:`URLLENGTH_LIMIT`.
|
|
||||||
|
|
||||||
Set by :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`.
|
|
||||||
|
|
|
||||||
|
|
@ -36,9 +36,9 @@ disable it if you want. For more information about the extension itself see
|
||||||
How to access the telnet console
|
How to access the telnet console
|
||||||
================================
|
================================
|
||||||
|
|
||||||
The telnet console listens on the first available TCP port from the range
|
The telnet console listens in the TCP port defined in the
|
||||||
defined in the :setting:`TELNETCONSOLE_PORT` setting, which defaults to
|
:setting:`TELNETCONSOLE_PORT` setting, which defaults to ``6023``. To access
|
||||||
``[6023, 6073]``. To access the console you need to type::
|
the console you need to type::
|
||||||
|
|
||||||
telnet localhost 6023
|
telnet localhost 6023
|
||||||
Trying localhost...
|
Trying localhost...
|
||||||
|
|
@ -107,8 +107,8 @@ Here are some example tasks you can do with the telnet console:
|
||||||
View engine status
|
View engine status
|
||||||
------------------
|
------------------
|
||||||
|
|
||||||
You can use the ``est()`` method provided by the console to quickly show the
|
You can use the ``est()`` method of the Scrapy engine to quickly show its state
|
||||||
engine status::
|
using the telnet console::
|
||||||
|
|
||||||
telnet localhost 6023
|
telnet localhost 6023
|
||||||
>>> est()
|
>>> est()
|
||||||
|
|
|
||||||
|
|
@ -39,8 +39,8 @@ API stability
|
||||||
|
|
||||||
API stability was one of the major goals for the *1.0* release.
|
API stability was one of the major goals for the *1.0* release.
|
||||||
|
|
||||||
Methods or functions that start with a single underscore (``_``) are private
|
Methods or functions that start with a single dash (``_``) are private and
|
||||||
and should never be relied upon as stable.
|
should never be relied as stable.
|
||||||
|
|
||||||
Also, keep in mind that stable doesn't mean complete: stable APIs could grow
|
Also, keep in mind that stable doesn't mean complete: stable APIs could grow
|
||||||
new methods or functionality but the existing methods should keep working the
|
new methods or functionality but the existing methods should keep working the
|
||||||
|
|
|
||||||
|
|
@ -60,25 +60,6 @@ Source = "https://github.com/scrapy/scrapy"
|
||||||
Tracker = "https://github.com/scrapy/scrapy/issues"
|
Tracker = "https://github.com/scrapy/scrapy/issues"
|
||||||
"Release notes" = "https://docs.scrapy.org/en/latest/news.html"
|
"Release notes" = "https://docs.scrapy.org/en/latest/news.html"
|
||||||
|
|
||||||
[project.optional-dependencies]
|
|
||||||
bpython = ["bpython>=0.7.1"]
|
|
||||||
brotli = [
|
|
||||||
"brotli>=1.2.0; implementation_name != 'pypy'",
|
|
||||||
"brotlicffi>=1.2.0.0; implementation_name == 'pypy'",
|
|
||||||
]
|
|
||||||
gcs = ["google-cloud-storage>=1.29.0"]
|
|
||||||
httpx = ["httpx2[http2,socks]>=2.0.0"]
|
|
||||||
images = ["Pillow>=8.3.2"]
|
|
||||||
ipython = ["ipython>=8.15.0"]
|
|
||||||
ptpython = ["ptpython>=3.0.23"]
|
|
||||||
robotparser = ["robotexclusionrulesparser>=1.6.2"]
|
|
||||||
s3 = ["boto3>=1.20.0"]
|
|
||||||
twisted-http2 = ["Twisted[http2]>=21.7.0"]
|
|
||||||
uvloop = [
|
|
||||||
"uvloop>=0.16.0; platform_system != 'Windows' and implementation_name != 'pypy'",
|
|
||||||
]
|
|
||||||
zstd = ["zstandard>=0.16.0; implementation_name != 'pypy'"]
|
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
scrapy = "scrapy.cmdline:execute"
|
scrapy = "scrapy.cmdline:execute"
|
||||||
|
|
||||||
|
|
@ -113,75 +94,7 @@ untyped_calls_exclude = [
|
||||||
[[tool.mypy.overrides]]
|
[[tool.mypy.overrides]]
|
||||||
module = "tests.*"
|
module = "tests.*"
|
||||||
allow_untyped_defs = true
|
allow_untyped_defs = true
|
||||||
allow_incomplete_defs = true # 59 errors
|
allow_incomplete_defs = true # 48 errors
|
||||||
|
|
||||||
# TODO
|
|
||||||
[[tool.mypy.overrides]]
|
|
||||||
module = [
|
|
||||||
"tests.mockserver.*",
|
|
||||||
"tests.spiders",
|
|
||||||
"tests.test_closespider",
|
|
||||||
"tests.test_cmdline",
|
|
||||||
"tests.test_contracts",
|
|
||||||
"tests.test_core_downloader",
|
|
||||||
"tests.test_downloader_handler_twisted_ftp",
|
|
||||||
"tests.test_downloadermiddleware_cookies",
|
|
||||||
"tests.test_downloadermiddleware_httpauth",
|
|
||||||
"tests.test_downloadermiddleware_httpcache",
|
|
||||||
"tests.test_downloadermiddleware_httpcompression",
|
|
||||||
"tests.test_downloadermiddleware_httpproxy",
|
|
||||||
"tests.test_downloadermiddleware_offsite",
|
|
||||||
"tests.test_downloadermiddleware_redirect",
|
|
||||||
"tests.test_downloadermiddleware_redirect_base",
|
|
||||||
"tests.test_downloadermiddleware_redirect_metarefresh",
|
|
||||||
"tests.test_downloadermiddleware_retry",
|
|
||||||
"tests.test_downloadermiddleware_robotstxt",
|
|
||||||
"tests.test_downloaderslotssettings",
|
|
||||||
"tests.test_dupefilters",
|
|
||||||
"tests.test_engine_loop",
|
|
||||||
"tests.test_exporters",
|
|
||||||
"tests.test_extension_statsmailer",
|
|
||||||
"tests.test_extension_throttle",
|
|
||||||
"tests.test_feedexport",
|
|
||||||
"tests.test_feedexport_postprocess",
|
|
||||||
"tests.test_feedexport_storages",
|
|
||||||
"tests.test_feedexport_uri_params",
|
|
||||||
"tests.test_http2_client_protocol",
|
|
||||||
"tests.test_http_headers",
|
|
||||||
"tests.test_http_request",
|
|
||||||
"tests.test_http_request_form",
|
|
||||||
"tests.test_http_response",
|
|
||||||
"tests.test_http_response_text",
|
|
||||||
"tests.test_item",
|
|
||||||
"tests.test_linkextractors",
|
|
||||||
"tests.test_loader",
|
|
||||||
"tests.test_logformatter",
|
|
||||||
"tests.test_mail",
|
|
||||||
"tests.test_pipeline_crawl",
|
|
||||||
"tests.test_pipeline_files",
|
|
||||||
"tests.test_pipeline_images",
|
|
||||||
"tests.test_pipeline_media",
|
|
||||||
"tests.test_pipelines",
|
|
||||||
"tests.test_pqueues",
|
|
||||||
"tests.test_request_attribute_binding",
|
|
||||||
"tests.test_request_cb_kwargs",
|
|
||||||
"tests.test_request_dict",
|
|
||||||
"tests.test_request_left",
|
|
||||||
"tests.test_robotstxt_interface",
|
|
||||||
"tests.test_scheduler_base",
|
|
||||||
"tests.test_settings",
|
|
||||||
"tests.test_spider",
|
|
||||||
"tests.test_spider_crawl",
|
|
||||||
"tests.test_spidermiddleware_output_chain",
|
|
||||||
"tests.test_spidermiddleware_process_start",
|
|
||||||
"tests.test_spider_sitemap",
|
|
||||||
"tests.test_squeues",
|
|
||||||
"tests.test_squeues_request",
|
|
||||||
"tests.test_stats",
|
|
||||||
"tests.utils.bases.http_request",
|
|
||||||
"tests.utils.bases.http_response",
|
|
||||||
"tests.utils.bases.spider",
|
|
||||||
]
|
|
||||||
check_untyped_defs = false
|
check_untyped_defs = false
|
||||||
|
|
||||||
# Interface classes are hard to support
|
# Interface classes are hard to support
|
||||||
|
|
@ -218,6 +131,7 @@ module = [
|
||||||
"pyftpdlib.*",
|
"pyftpdlib.*",
|
||||||
"pytest_twisted",
|
"pytest_twisted",
|
||||||
"robotexclusionrulesparser",
|
"robotexclusionrulesparser",
|
||||||
|
"testfixtures",
|
||||||
"zope.interface.*",
|
"zope.interface.*",
|
||||||
]
|
]
|
||||||
ignore_missing_imports = true
|
ignore_missing_imports = true
|
||||||
|
|
@ -355,7 +269,7 @@ markers = [
|
||||||
"requires_uvloop: marks tests as only enabled when uvloop is known to be working",
|
"requires_uvloop: marks tests as only enabled when uvloop is known to be working",
|
||||||
"requires_botocore: marks tests that need botocore (but not boto3)",
|
"requires_botocore: marks tests that need botocore (but not boto3)",
|
||||||
"requires_boto3: marks tests that need botocore and boto3",
|
"requires_boto3: marks tests that need botocore and boto3",
|
||||||
"requires_mitmproxy: marks tests that need a mitmdump executable",
|
"requires_mitmproxy: marks tests that need mitmproxy",
|
||||||
"requires_internet: marks tests that need real Internet access",
|
"requires_internet: marks tests that need real Internet access",
|
||||||
]
|
]
|
||||||
filterwarnings = [
|
filterwarnings = [
|
||||||
|
|
|
||||||
|
|
@ -225,11 +225,13 @@ def _run_command(cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace)
|
||||||
def _run_command_profiled(
|
def _run_command_profiled(
|
||||||
cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace
|
cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace
|
||||||
) -> None:
|
) -> None:
|
||||||
sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
|
if opts.profile:
|
||||||
|
sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
|
||||||
loc = locals()
|
loc = locals()
|
||||||
p = cProfile.Profile()
|
p = cProfile.Profile()
|
||||||
p.runctx("cmd.run(args, opts)", globals(), loc)
|
p.runctx("cmd.run(args, opts)", globals(), loc)
|
||||||
p.dump_stats(opts.profile)
|
if opts.profile:
|
||||||
|
p.dump_stats(opts.profile)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand):
|
||||||
return "[options] <spider>"
|
return "[options] <spider>"
|
||||||
|
|
||||||
def short_desc(self) -> str:
|
def short_desc(self) -> str:
|
||||||
return "Run a spider of the current project, by name"
|
return "Run a spider"
|
||||||
|
|
||||||
def run(self, args: list[str], opts: argparse.Namespace) -> None:
|
def run(self, args: list[str], opts: argparse.Namespace) -> None:
|
||||||
if len(args) < 1:
|
if len(args) < 1:
|
||||||
|
|
|
||||||
|
|
@ -32,7 +32,10 @@ def sanitize_module_name(module_name: str) -> str:
|
||||||
|
|
||||||
def extract_domain(url: str) -> str:
|
def extract_domain(url: str) -> str:
|
||||||
"""Extract domain name from URL string"""
|
"""Extract domain name from URL string"""
|
||||||
return urlparse(url).netloc
|
o = urlparse(url)
|
||||||
|
if o.scheme == "" and o.netloc == "":
|
||||||
|
o = urlparse("//" + url.lstrip("/"))
|
||||||
|
return o.netloc
|
||||||
|
|
||||||
|
|
||||||
def verify_url_scheme(url: str) -> str:
|
def verify_url_scheme(url: str) -> str:
|
||||||
|
|
|
||||||
|
|
@ -41,7 +41,7 @@ class Command(BaseRunSpiderCommand):
|
||||||
spider: Spider | None = None
|
spider: Spider | None = None
|
||||||
items: ClassVar[dict[int, list[Any]]] = {}
|
items: ClassVar[dict[int, list[Any]]] = {}
|
||||||
requests: ClassVar[dict[int, list[Request]]] = {}
|
requests: ClassVar[dict[int, list[Request]]] = {}
|
||||||
spidercls: type[Spider] | None = None
|
spidercls: type[Spider] | None
|
||||||
|
|
||||||
first_response = None
|
first_response = None
|
||||||
|
|
||||||
|
|
@ -346,8 +346,6 @@ class Command(BaseRunSpiderCommand):
|
||||||
self.first_response = response
|
self.first_response = response
|
||||||
|
|
||||||
cb = self._get_callback(spider=spider, opts=opts, response=response)
|
cb = self._get_callback(spider=spider, opts=opts, response=response)
|
||||||
assert response.request
|
|
||||||
response.request.callback = cb
|
|
||||||
|
|
||||||
# parse items and requests
|
# parse items and requests
|
||||||
depth: int = response.meta["_depth"]
|
depth: int = response.meta["_depth"]
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,7 @@ class Command(BaseRunSpiderCommand):
|
||||||
return "[options] <spider_file>"
|
return "[options] <spider_file>"
|
||||||
|
|
||||||
def short_desc(self) -> str:
|
def short_desc(self) -> str:
|
||||||
return "Run a spider from a Python file, no project required"
|
return "Run a self-contained spider (without creating a project)"
|
||||||
|
|
||||||
def long_desc(self) -> str:
|
def long_desc(self) -> str:
|
||||||
return "Run the spider defined in the given file"
|
return "Run the spider defined in the given file"
|
||||||
|
|
|
||||||
|
|
@ -22,16 +22,7 @@ if TYPE_CHECKING:
|
||||||
|
|
||||||
|
|
||||||
class Contract:
|
class Contract:
|
||||||
"""Base class for :ref:`custom contracts <topics-contracts>`.
|
"""Abstract class for contracts"""
|
||||||
|
|
||||||
*method* is the callback function to which the contract is associated.
|
|
||||||
|
|
||||||
*args* is the list of arguments passed into the docstring, separated by
|
|
||||||
whitespace.
|
|
||||||
|
|
||||||
Subclasses may override :meth:`adjust_request_args`, and define a
|
|
||||||
``pre_process`` method or a ``post_process`` method, or both.
|
|
||||||
"""
|
|
||||||
|
|
||||||
request_cls: type[Request] | None = None
|
request_cls: type[Request] | None = None
|
||||||
name: str
|
name: str
|
||||||
|
|
@ -99,13 +90,6 @@ class Contract:
|
||||||
return request
|
return request
|
||||||
|
|
||||||
def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]:
|
def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]:
|
||||||
"""Receive a ``dict`` with the default arguments for the sample request
|
|
||||||
and return it, either unmodified or with changes.
|
|
||||||
|
|
||||||
:class:`~scrapy.Request` is used by default, but this can be changed
|
|
||||||
with the ``request_cls`` attribute. If multiple contracts in the chain
|
|
||||||
define this attribute, the last one is used.
|
|
||||||
"""
|
|
||||||
return args
|
return args
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -15,15 +15,8 @@ if TYPE_CHECKING:
|
||||||
|
|
||||||
# contracts
|
# contracts
|
||||||
class UrlContract(Contract):
|
class UrlContract(Contract):
|
||||||
"""Sets (``@url``) the sample URL used when checking the other contract
|
"""Contract to set the url of the request (mandatory)
|
||||||
conditions of a callback.
|
@url http://scrapy.org
|
||||||
|
|
||||||
This contract is mandatory: callbacks lacking it are ignored when running
|
|
||||||
the checks.
|
|
||||||
|
|
||||||
.. code-block:: none
|
|
||||||
|
|
||||||
@url url
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name = "url"
|
name = "url"
|
||||||
|
|
@ -34,14 +27,10 @@ class UrlContract(Contract):
|
||||||
|
|
||||||
|
|
||||||
class CallbackKeywordArgumentsContract(Contract):
|
class CallbackKeywordArgumentsContract(Contract):
|
||||||
"""Sets (``@cb_kwargs``) the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`
|
"""Contract to set the keyword arguments for the request.
|
||||||
attribute of the sample request.
|
The value should be a JSON-encoded dictionary, e.g.:
|
||||||
|
|
||||||
Its value must be a valid JSON dictionary.
|
@cb_kwargs {"arg1": "some value"}
|
||||||
|
|
||||||
.. code-block:: none
|
|
||||||
|
|
||||||
@cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name = "cb_kwargs"
|
name = "cb_kwargs"
|
||||||
|
|
@ -52,14 +41,10 @@ class CallbackKeywordArgumentsContract(Contract):
|
||||||
|
|
||||||
|
|
||||||
class MetadataContract(Contract):
|
class MetadataContract(Contract):
|
||||||
"""Sets (``@meta``) the :attr:`meta <scrapy.Request.meta>` attribute of the
|
"""Contract to set metadata arguments for the request.
|
||||||
sample request.
|
The value should be JSON-encoded dictionary, e.g.:
|
||||||
|
|
||||||
Its value must be a valid JSON dictionary.
|
@meta {"arg1": "some value"}
|
||||||
|
|
||||||
.. code-block:: none
|
|
||||||
|
|
||||||
@meta {"arg1": "value1", "arg2": "value2", ...}
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name = "meta"
|
name = "meta"
|
||||||
|
|
@ -70,29 +55,16 @@ class MetadataContract(Contract):
|
||||||
|
|
||||||
|
|
||||||
class ReturnsContract(Contract):
|
class ReturnsContract(Contract):
|
||||||
"""Sets (``@returns``) lower and upper bounds for the items and requests
|
"""Contract to check the output of a callback
|
||||||
returned by a callback.
|
|
||||||
|
|
||||||
The upper bound is optional:
|
general form:
|
||||||
|
@returns request(s)/item(s) [min=1 [max]]
|
||||||
|
|
||||||
.. code-block:: none
|
e.g.:
|
||||||
|
@returns request
|
||||||
@returns item(s)|request(s) [min [max]]
|
@returns request 2
|
||||||
|
@returns request 2 10
|
||||||
For example:
|
@returns request 0 10
|
||||||
|
|
||||||
.. code-block:: none
|
|
||||||
|
|
||||||
@returns request
|
|
||||||
@returns request 2
|
|
||||||
@returns request 2 10
|
|
||||||
@returns request 0 10
|
|
||||||
|
|
||||||
Set both bounds to the same value to require an exact number:
|
|
||||||
|
|
||||||
.. code-block:: none
|
|
||||||
|
|
||||||
@returns request 2 2
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name = "returns"
|
name = "returns"
|
||||||
|
|
@ -143,12 +115,8 @@ class ReturnsContract(Contract):
|
||||||
|
|
||||||
|
|
||||||
class ScrapesContract(Contract):
|
class ScrapesContract(Contract):
|
||||||
"""Checks (``@scrapes``) that all items returned by a callback have the
|
"""Contract to check presence of fields in scraped items
|
||||||
specified fields.
|
@scrapes page_name page_body
|
||||||
|
|
||||||
.. code-block:: none
|
|
||||||
|
|
||||||
@scrapes field_1 field_2 ...
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name = "scrapes"
|
name = "scrapes"
|
||||||
|
|
|
||||||
|
|
@ -138,8 +138,7 @@ class Downloader:
|
||||||
self.active.remove(request)
|
self.active.remove(request)
|
||||||
|
|
||||||
def needs_backout(self) -> bool:
|
def needs_backout(self) -> bool:
|
||||||
# A total concurrency of 0 means no limit.
|
return len(self.active) >= self.total_concurrency
|
||||||
return 0 < self.total_concurrency <= len(self.active)
|
|
||||||
|
|
||||||
@_warn_spider_arg
|
@_warn_spider_arg
|
||||||
def _get_slot(
|
def _get_slot(
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ from twisted.internet.ssl import (
|
||||||
from twisted.web.client import BrowserLikePolicyForHTTPS
|
from twisted.web.client import BrowserLikePolicyForHTTPS
|
||||||
from twisted.web.iweb import IPolicyForHTTPS
|
from twisted.web.iweb import IPolicyForHTTPS
|
||||||
from zope.interface.declarations import implementer
|
from zope.interface.declarations import implementer
|
||||||
|
from zope.interface.verify import verifyObject
|
||||||
|
|
||||||
from scrapy.core.downloader.tls import (
|
from scrapy.core.downloader.tls import (
|
||||||
_TWISTED_VERSION_MAP,
|
_TWISTED_VERSION_MAP,
|
||||||
|
|
@ -231,6 +232,7 @@ class _AcceptableProtocolsContextFactory:
|
||||||
# all of this with _ScrapyClientContextFactory.acceptableProtocols.
|
# all of this with _ScrapyClientContextFactory.acceptableProtocols.
|
||||||
|
|
||||||
def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]):
|
def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]):
|
||||||
|
verifyObject(IPolicyForHTTPS, context_factory)
|
||||||
self._wrapped_context_factory: Any = context_factory
|
self._wrapped_context_factory: Any = context_factory
|
||||||
self._acceptable_protocols: list[bytes] = acceptable_protocols
|
self._acceptable_protocols: list[bytes] = acceptable_protocols
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -30,7 +30,7 @@ from ._base_streaming import BaseStreamingDownloadHandler, _BaseResponseArgs
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import AsyncIterator
|
from collections.abc import AsyncIterator
|
||||||
|
|
||||||
from httpcore2 import AsyncNetworkStream
|
from httpcore import AsyncNetworkStream
|
||||||
|
|
||||||
from scrapy import Request
|
from scrapy import Request
|
||||||
from scrapy.crawler import Crawler
|
from scrapy.crawler import Crawler
|
||||||
|
|
@ -39,11 +39,8 @@ if TYPE_CHECKING:
|
||||||
HAS_SOCKS = HAS_HTTP2 = False
|
HAS_SOCKS = HAS_HTTP2 = False
|
||||||
|
|
||||||
try:
|
try:
|
||||||
try:
|
import httpx
|
||||||
import httpx2 as httpx
|
except ImportError:
|
||||||
except ImportError: # pragma: no cover
|
|
||||||
import httpx # type: ignore[import-not-found,no-redef]
|
|
||||||
except ImportError: # pragma: no cover
|
|
||||||
httpx = None # type: ignore[assignment]
|
httpx = None # type: ignore[assignment]
|
||||||
else:
|
else:
|
||||||
# a small hack to avoid importing these optional extras unconditionally
|
# a small hack to avoid importing these optional extras unconditionally
|
||||||
|
|
@ -87,20 +84,19 @@ class HttpxDownloadHandler(_Base):
|
||||||
self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED")
|
self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED")
|
||||||
if self._enable_h2 and not HAS_HTTP2: # pragma: no cover
|
if self._enable_h2 and not HAS_HTTP2: # pragma: no cover
|
||||||
raise NotConfigured(
|
raise NotConfigured(
|
||||||
f"HTTP/2 support in {type(self).__name__} requires the 'httpx2[http2]' extra to be installed."
|
f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed."
|
||||||
)
|
)
|
||||||
self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
|
self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
|
||||||
self._bind_host: str | None = self._get_bind_address_host()
|
self._bind_host: str | None = self._get_bind_address_host()
|
||||||
self._limits: httpx.Limits = httpx.Limits(
|
self._limits: httpx.Limits = httpx.Limits(
|
||||||
# hard limit on simultaneous connections (None for no limit, which
|
# hard limit on simultaneous connections
|
||||||
# is what a CONCURRENT_REQUESTS of 0 means)
|
max_connections=self._pool_size_total,
|
||||||
max_connections=self._pool_size_total or None,
|
|
||||||
# total number of idle connections in the pool (extra ones are closed)
|
# total number of idle connections in the pool (extra ones are closed)
|
||||||
max_keepalive_connections=self._pool_size_total or None,
|
max_keepalive_connections=self._pool_size_total,
|
||||||
)
|
)
|
||||||
|
|
||||||
self._default_client: httpx.AsyncClient = self._make_client()
|
self._default_client: httpx.AsyncClient = self._make_client()
|
||||||
# httpx2 doesn't support per-request proxies: https://github.com/pydantic/httpx2/issues/818,
|
# httpx doesn't support per-request proxies: https://github.com/encode/httpx/discussions/3183,
|
||||||
# so we keep a pool of clients per proxy URL. LRU eviction can be added here if needed.
|
# so we keep a pool of clients per proxy URL. LRU eviction can be added here if needed.
|
||||||
self._proxy_clients: dict[str, httpx.AsyncClient] = {}
|
self._proxy_clients: dict[str, httpx.AsyncClient] = {}
|
||||||
|
|
||||||
|
|
@ -108,7 +104,7 @@ class HttpxDownloadHandler(_Base):
|
||||||
def _check_deps_installed() -> None:
|
def _check_deps_installed() -> None:
|
||||||
if httpx is None: # pragma: no cover
|
if httpx is None: # pragma: no cover
|
||||||
raise NotConfigured(
|
raise NotConfigured(
|
||||||
"HttpxDownloadHandler requires the httpx2 library to be installed."
|
"HttpxDownloadHandler requires the httpx library to be installed."
|
||||||
)
|
)
|
||||||
|
|
||||||
def _make_client(self, proxy_url: str | None = None) -> httpx.AsyncClient:
|
def _make_client(self, proxy_url: str | None = None) -> httpx.AsyncClient:
|
||||||
|
|
@ -132,7 +128,7 @@ class HttpxDownloadHandler(_Base):
|
||||||
proxy=proxy,
|
proxy=proxy,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
# https://github.com/pydantic/httpx2/issues/368
|
# https://github.com/encode/httpx/discussions/1566
|
||||||
for header_name in ("accept", "accept-encoding", "user-agent"):
|
for header_name in ("accept", "accept-encoding", "user-agent"):
|
||||||
client.headers.pop(header_name, None)
|
client.headers.pop(header_name, None)
|
||||||
return client
|
return client
|
||||||
|
|
@ -153,7 +149,7 @@ class HttpxDownloadHandler(_Base):
|
||||||
proxy = self._extract_proxy_url_with_creds(request)
|
proxy = self._extract_proxy_url_with_creds(request)
|
||||||
if proxy and proxy.startswith("socks") and not HAS_SOCKS: # pragma: no cover
|
if proxy and proxy.startswith("socks") and not HAS_SOCKS: # pragma: no cover
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"SOCKS proxy support in {type(self).__name__} requires the 'httpx2[socks]' extra to be installed."
|
f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed."
|
||||||
)
|
)
|
||||||
client = self._get_client(proxy)
|
client = self._get_client(proxy)
|
||||||
headers = self._request_headers(request).to_tuple_list()
|
headers = self._request_headers(request).to_tuple_list()
|
||||||
|
|
@ -179,7 +175,7 @@ class HttpxDownloadHandler(_Base):
|
||||||
raise DownloadConnectionRefusedError(str(e)) from e
|
raise DownloadConnectionRefusedError(str(e)) from e
|
||||||
except httpx.ProxyError as e:
|
except httpx.ProxyError as e:
|
||||||
raise DownloadConnectionRefusedError(str(e)) from e
|
raise DownloadConnectionRefusedError(str(e)) from e
|
||||||
except DOWNLOAD_FAILED_EXCEPTIONS as e: # pylint: disable=catching-non-exception
|
except DOWNLOAD_FAILED_EXCEPTIONS as e:
|
||||||
raise DownloadFailedError(str(e)) from e
|
raise DownloadFailedError(str(e)) from e
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|
@ -222,7 +218,7 @@ class HttpxDownloadHandler(_Base):
|
||||||
def _log_tls_info(self, response: httpx.Response, request: Request) -> None:
|
def _log_tls_info(self, response: httpx.Response, request: Request) -> None:
|
||||||
network_stream: AsyncNetworkStream = response.extensions["network_stream"]
|
network_stream: AsyncNetworkStream = response.extensions["network_stream"]
|
||||||
extra_ssl_object = network_stream.get_extra_info("ssl_object")
|
extra_ssl_object = network_stream.get_extra_info("ssl_object")
|
||||||
if isinstance(extra_ssl_object, ssl.SSLObject): # pragma: no branch
|
if isinstance(extra_ssl_object, ssl.SSLObject):
|
||||||
_log_sslobj_debug_info(extra_ssl_object)
|
_log_sslobj_debug_info(extra_ssl_object)
|
||||||
|
|
||||||
async def close(self) -> None:
|
async def close(self) -> None:
|
||||||
|
|
|
||||||
|
|
@ -126,4 +126,5 @@ class FTPDownloadHandler(BaseDownloadHandler):
|
||||||
headers = {"local filename": protocol.filename or b"", "size": protocol.size}
|
headers = {"local filename": protocol.filename or b"", "size": protocol.size}
|
||||||
body = protocol.filename or protocol.body.read()
|
body = protocol.filename or protocol.body.read()
|
||||||
respcls = responsetypes.from_args(url=request.url, body=body)
|
respcls = responsetypes.from_args(url=request.url, body=body)
|
||||||
return respcls(url=request.url, status=200, body=body, headers=headers)
|
# hints for Headers-related types may need to be fixed to not use AnyStr
|
||||||
|
return respcls(url=request.url, status=200, body=body, headers=headers) # type: ignore[arg-type]
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,9 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import warnings
|
|
||||||
from typing import TYPE_CHECKING, Any, cast
|
from typing import TYPE_CHECKING, Any, cast
|
||||||
|
|
||||||
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
|
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
|
||||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
from scrapy.exceptions import NotConfigured
|
||||||
from scrapy.utils.boto import is_botocore_available
|
from scrapy.utils.boto import is_botocore_available
|
||||||
from scrapy.utils.httpobj import urlparse_cached
|
from scrapy.utils.httpobj import urlparse_cached
|
||||||
from scrapy.utils.misc import build_from_crawler, load_object
|
from scrapy.utils.misc import build_from_crawler, load_object
|
||||||
|
|
@ -50,16 +49,7 @@ class S3DownloadHandler(BaseDownloadHandler):
|
||||||
|
|
||||||
async def download_request(self, request: Request) -> Response:
|
async def download_request(self, request: Request) -> Response:
|
||||||
p = urlparse_cached(request)
|
p = urlparse_cached(request)
|
||||||
if request.meta.get("is_secure") is False:
|
scheme = "http" if request.meta.get("is_secure") is False else "https"
|
||||||
warnings.warn(
|
|
||||||
"Passing is_secure=False for s3:// requests is deprecated."
|
|
||||||
" In future Scrapy releases this flag will be ignored.",
|
|
||||||
ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
scheme = "http"
|
|
||||||
else:
|
|
||||||
scheme = "https"
|
|
||||||
bucket = p.hostname
|
bucket = p.hostname
|
||||||
path = p.path + "?" + p.query if p.query else p.path
|
path = p.path + "?" + p.query if p.query else p.path
|
||||||
url = f"{scheme}://{bucket}.s3.amazonaws.com{path}"
|
url = f"{scheme}://{bucket}.s3.amazonaws.com{path}"
|
||||||
|
|
|
||||||
|
|
@ -470,17 +470,16 @@ class ExecutionEngine:
|
||||||
"""
|
"""
|
||||||
if self.spider is None:
|
if self.spider is None:
|
||||||
raise RuntimeError(f"No open spider to crawl: {request}")
|
raise RuntimeError(f"No open spider to crawl: {request}")
|
||||||
while True:
|
try:
|
||||||
try:
|
response_or_request = await maybe_deferred_to_future(
|
||||||
response_or_request = await maybe_deferred_to_future(
|
self._download(request)
|
||||||
self._download(request)
|
)
|
||||||
)
|
finally:
|
||||||
finally:
|
assert self._slot is not None
|
||||||
assert self._slot is not None
|
self._slot.remove_request(request)
|
||||||
self._slot.remove_request(request)
|
if isinstance(response_or_request, Request):
|
||||||
if not isinstance(response_or_request, Request):
|
return await self.download_async(response_or_request)
|
||||||
return response_or_request
|
return response_or_request
|
||||||
request = response_or_request
|
|
||||||
|
|
||||||
@inlineCallbacks
|
@inlineCallbacks
|
||||||
def _download(
|
def _download(
|
||||||
|
|
|
||||||
|
|
@ -315,7 +315,7 @@ class Stream:
|
||||||
0, self.metadata["remaining_content_length"]
|
0, self.metadata["remaining_content_length"]
|
||||||
)
|
)
|
||||||
|
|
||||||
# End the stream if no more data needs to be sent
|
# End the stream if no more data needs to be send
|
||||||
if self.metadata["remaining_content_length"] == 0:
|
if self.metadata["remaining_content_length"] == 0:
|
||||||
self._protocol.conn.end_stream(self.stream_id)
|
self._protocol.conn.end_stream(self.stream_id)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -289,11 +289,11 @@ class Scheduler(BaseScheduler):
|
||||||
|
|
||||||
:param dqclass: A class to be used as persistent request queue.
|
:param dqclass: A class to be used as persistent request queue.
|
||||||
The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default.
|
The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default.
|
||||||
:type dqclass: type
|
:type dqclass: class
|
||||||
|
|
||||||
:param mqclass: A class to be used as non-persistent request queue.
|
:param mqclass: A class to be used as non-persistent request queue.
|
||||||
The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default.
|
The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default.
|
||||||
:type mqclass: type
|
:type mqclass: class
|
||||||
|
|
||||||
:param logunser: A boolean that indicates whether or not unserializable requests should be logged.
|
:param logunser: A boolean that indicates whether or not unserializable requests should be logged.
|
||||||
The value for the :setting:`SCHEDULER_DEBUG` setting is used by default.
|
The value for the :setting:`SCHEDULER_DEBUG` setting is used by default.
|
||||||
|
|
@ -306,7 +306,7 @@ class Scheduler(BaseScheduler):
|
||||||
|
|
||||||
:param pqclass: A class to be used as priority queue for requests.
|
:param pqclass: A class to be used as priority queue for requests.
|
||||||
The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default.
|
The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default.
|
||||||
:type pqclass: type
|
:type pqclass: class
|
||||||
|
|
||||||
:param crawler: The crawler object corresponding to the current crawl.
|
:param crawler: The crawler object corresponding to the current crawl.
|
||||||
:type crawler: :class:`scrapy.crawler.Crawler`
|
:type crawler: :class:`scrapy.crawler.Crawler`
|
||||||
|
|
@ -366,8 +366,8 @@ class Scheduler(BaseScheduler):
|
||||||
Unless the received request is filtered out by the Dupefilter, attempt to push
|
Unless the received request is filtered out by the Dupefilter, attempt to push
|
||||||
it into the disk queue, falling back to pushing it into the memory queue.
|
it into the disk queue, falling back to pushing it into the memory queue.
|
||||||
|
|
||||||
Increment the appropriate stats, such as: :stat:`scheduler/enqueued`,
|
Increment the appropriate stats, such as: ``scheduler/enqueued``,
|
||||||
:stat:`scheduler/enqueued/disk`, :stat:`scheduler/enqueued/memory`.
|
``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``.
|
||||||
|
|
||||||
Return ``True`` if the request was stored successfully, ``False`` otherwise.
|
Return ``True`` if the request was stored successfully, ``False`` otherwise.
|
||||||
"""
|
"""
|
||||||
|
|
@ -390,8 +390,8 @@ class Scheduler(BaseScheduler):
|
||||||
falling back to the disk queue if the memory queue is empty.
|
falling back to the disk queue if the memory queue is empty.
|
||||||
Return ``None`` if there are no more enqueued requests.
|
Return ``None`` if there are no more enqueued requests.
|
||||||
|
|
||||||
Increment the appropriate stats, such as: :stat:`scheduler/dequeued`,
|
Increment the appropriate stats, such as: ``scheduler/dequeued``,
|
||||||
:stat:`scheduler/dequeued/disk`, :stat:`scheduler/dequeued/memory`.
|
``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``.
|
||||||
"""
|
"""
|
||||||
request: Request | None = self.mqs.pop()
|
request: Request | None = self.mqs.pop()
|
||||||
assert self.stats is not None
|
assert self.stats is not None
|
||||||
|
|
|
||||||
|
|
@ -39,11 +39,7 @@ from scrapy.utils.reactor import (
|
||||||
verify_installed_asyncio_event_loop,
|
verify_installed_asyncio_event_loop,
|
||||||
verify_installed_reactor,
|
verify_installed_reactor,
|
||||||
)
|
)
|
||||||
from scrapy.utils.reactorless import (
|
from scrapy.utils.reactorless import install_reactor_import_hook
|
||||||
ReactorImportHook,
|
|
||||||
install_reactor_import_hook,
|
|
||||||
uninstall_reactor_import_hook,
|
|
||||||
)
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from collections.abc import Awaitable, Generator, Iterable
|
from collections.abc import Awaitable, Generator, Iterable
|
||||||
|
|
@ -841,7 +837,6 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
|
||||||
super().__init__(settings, install_root_handler)
|
super().__init__(settings, install_root_handler)
|
||||||
logger.debug("Using AsyncCrawlerProcess")
|
logger.debug("Using AsyncCrawlerProcess")
|
||||||
self._reactorless_loop: asyncio.AbstractEventLoop | None = None
|
self._reactorless_loop: asyncio.AbstractEventLoop | None = None
|
||||||
self._reactor_import_hook: ReactorImportHook | None = None
|
|
||||||
# We want the asyncio event loop to be installed early, so that it's
|
# We want the asyncio event loop to be installed early, so that it's
|
||||||
# always the correct one. And as we do that, we can also install the
|
# always the correct one. And as we do that, we can also install the
|
||||||
# reactor here.
|
# reactor here.
|
||||||
|
|
@ -854,7 +849,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
|
||||||
"TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed."
|
"TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed."
|
||||||
)
|
)
|
||||||
self._reactorless_loop = set_asyncio_event_loop(loop_path)
|
self._reactorless_loop = set_asyncio_event_loop(loop_path)
|
||||||
self._reactor_import_hook = install_reactor_import_hook()
|
install_reactor_import_hook()
|
||||||
elif is_reactor_installed():
|
elif is_reactor_installed():
|
||||||
# The user could install a reactor before this class is instantiated.
|
# The user could install a reactor before this class is instantiated.
|
||||||
# We need to make sure the reactor is the correct one and the loop
|
# We need to make sure the reactor is the correct one and the loop
|
||||||
|
|
@ -984,10 +979,6 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
|
||||||
loop.run_until_complete(loop.shutdown_asyncgens())
|
loop.run_until_complete(loop.shutdown_asyncgens())
|
||||||
loop.run_until_complete(loop.shutdown_default_executor())
|
loop.run_until_complete(loop.shutdown_default_executor())
|
||||||
finally:
|
finally:
|
||||||
# loop.close() can raise, so we uninstall the hook first
|
|
||||||
if self._reactor_import_hook: # pragma: no branch
|
|
||||||
uninstall_reactor_import_hook(self._reactor_import_hook)
|
|
||||||
self._reactor_import_hook = None
|
|
||||||
self._reactorless_main_task = None
|
self._reactorless_main_task = None
|
||||||
asyncio.set_event_loop(None)
|
asyncio.set_event_loop(None)
|
||||||
loop.close()
|
loop.close()
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,5 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
|
||||||
from email.utils import formatdate
|
from email.utils import formatdate
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
|
@ -29,9 +28,6 @@ if TYPE_CHECKING:
|
||||||
from scrapy.statscollectors import StatsCollector
|
from scrapy.statscollectors import StatsCollector
|
||||||
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
class HttpCacheMiddleware:
|
class HttpCacheMiddleware:
|
||||||
DOWNLOAD_EXCEPTIONS = (
|
DOWNLOAD_EXCEPTIONS = (
|
||||||
ConnectionDone,
|
ConnectionDone,
|
||||||
|
|
@ -81,20 +77,9 @@ class HttpCacheMiddleware:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Look for cached response and check if expired
|
# Look for cached response and check if expired
|
||||||
cachedresponse: Response | None
|
cachedresponse: Response | None = self.storage.retrieve_response(
|
||||||
try:
|
self.crawler.spider, request
|
||||||
cachedresponse = self.storage.retrieve_response(
|
)
|
||||||
self.crawler.spider, request
|
|
||||||
)
|
|
||||||
except Exception:
|
|
||||||
self.stats.inc_value("httpcache/retrieve_error")
|
|
||||||
logger.warning(
|
|
||||||
f"Could not read the cache entry for {request}, treating it as a "
|
|
||||||
f"cache miss.",
|
|
||||||
exc_info=True,
|
|
||||||
extra={"spider": self.crawler.spider},
|
|
||||||
)
|
|
||||||
cachedresponse = None
|
|
||||||
if cachedresponse is None:
|
if cachedresponse is None:
|
||||||
self.stats.inc_value("httpcache/miss")
|
self.stats.inc_value("httpcache/miss")
|
||||||
if self.ignore_missing:
|
if self.ignore_missing:
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,6 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import warnings
|
import warnings
|
||||||
from importlib.util import find_spec
|
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
from logging import getLogger
|
from logging import getLogger
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
@ -52,7 +51,11 @@ else:
|
||||||
else:
|
else:
|
||||||
ACCEPTED_ENCODINGS.append(b"br")
|
ACCEPTED_ENCODINGS.append(b"br")
|
||||||
|
|
||||||
if find_spec("zstandard") is not None:
|
try:
|
||||||
|
import zstandard # noqa: F401
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
ACCEPTED_ENCODINGS.append(b"zstd")
|
ACCEPTED_ENCODINGS.append(b"zstd")
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -120,14 +123,12 @@ class HttpCompressionMiddleware:
|
||||||
response.body, content_encoding, max_size
|
response.body, content_encoding, max_size
|
||||||
)
|
)
|
||||||
except _DecompressionMaxSizeExceeded as e:
|
except _DecompressionMaxSizeExceeded as e:
|
||||||
msg = (
|
raise IgnoreRequest(
|
||||||
f"Ignored response {response} because its body "
|
f"Ignored response {response} because its body "
|
||||||
f"({len(response.body)} B compressed, "
|
f"({len(response.body)} B compressed, "
|
||||||
f"{e.decompressed_size} B decompressed so far) exceeded "
|
f"{e.decompressed_size} B decompressed so far) exceeded "
|
||||||
f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
|
f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
|
||||||
)
|
) from e
|
||||||
logger.warning(msg)
|
|
||||||
raise IgnoreRequest(msg) from e
|
|
||||||
if len(response.body) < warn_size <= len(decoded_body):
|
if len(response.body) < warn_size <= len(decoded_body):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f"{response} body size after decompression "
|
f"{response} body size after decompression "
|
||||||
|
|
|
||||||
|
|
@ -61,7 +61,7 @@ class OffsiteMiddleware:
|
||||||
)
|
)
|
||||||
self.stats.inc_value("offsite/domains")
|
self.stats.inc_value("offsite/domains")
|
||||||
self.stats.inc_value("offsite/filtered")
|
self.stats.inc_value("offsite/filtered")
|
||||||
raise IgnoreRequest(f"Filtered offsite request to {domain!r}")
|
raise IgnoreRequest
|
||||||
|
|
||||||
def should_follow(self, request: Request, spider: Spider) -> bool:
|
def should_follow(self, request: Request, spider: Spider) -> bool:
|
||||||
regex = self.host_regex
|
regex = self.host_regex
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,6 @@ from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from twisted.internet.defer import Deferred
|
from twisted.internet.defer import Deferred
|
||||||
|
|
||||||
from scrapy import signals
|
|
||||||
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
from scrapy.exceptions import IgnoreRequest, NotConfigured
|
||||||
from scrapy.http import Request, Response
|
from scrapy.http import Request, Response
|
||||||
from scrapy.http.request import NO_CALLBACK
|
from scrapy.http.request import NO_CALLBACK
|
||||||
|
|
@ -99,7 +98,7 @@ class RobotsTxtMiddleware:
|
||||||
assert self.crawler.stats
|
assert self.crawler.stats
|
||||||
try:
|
try:
|
||||||
resp = await self.crawler.engine.download_async(robotsreq)
|
resp = await self.crawler.engine.download_async(robotsreq)
|
||||||
await self._parse_robots(resp, netloc, request)
|
self._parse_robots(resp, netloc)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
if not isinstance(e, IgnoreRequest):
|
if not isinstance(e, IgnoreRequest):
|
||||||
logger.error(
|
logger.error(
|
||||||
|
|
@ -116,20 +115,13 @@ class RobotsTxtMiddleware:
|
||||||
return await maybe_deferred_to_future(parser)
|
return await maybe_deferred_to_future(parser)
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
async def _parse_robots(
|
def _parse_robots(self, response: Response, netloc: str) -> None:
|
||||||
self, response: Response, netloc: str, request: Request
|
|
||||||
) -> None:
|
|
||||||
assert self.crawler.stats
|
assert self.crawler.stats
|
||||||
self.crawler.stats.inc_value("robotstxt/response_count")
|
self.crawler.stats.inc_value("robotstxt/response_count")
|
||||||
self.crawler.stats.inc_value(
|
self.crawler.stats.inc_value(
|
||||||
f"robotstxt/response_status_count/{response.status}"
|
f"robotstxt/response_status_count/{response.status}"
|
||||||
)
|
)
|
||||||
rp = self._parserimpl.from_crawler(self.crawler, response.body)
|
rp = self._parserimpl.from_crawler(self.crawler, response.body)
|
||||||
await self.crawler.signals.send_catch_log_async(
|
|
||||||
signal=signals.robots_parsed,
|
|
||||||
robotparser=rp,
|
|
||||||
request=request,
|
|
||||||
)
|
|
||||||
rp_dfd = self._parsers[netloc]
|
rp_dfd = self._parsers[netloc]
|
||||||
assert isinstance(rp_dfd, Deferred)
|
assert isinstance(rp_dfd, Deferred)
|
||||||
self._parsers[netloc] = rp
|
self._parsers[netloc] = rp
|
||||||
|
|
|
||||||
|
|
@ -16,15 +16,7 @@ if TYPE_CHECKING:
|
||||||
|
|
||||||
|
|
||||||
class NotConfigured(Exception):
|
class NotConfigured(Exception):
|
||||||
"""Raised by a :ref:`component <topics-components>` from its ``__init__()``
|
"""Indicates a missing configuration situation"""
|
||||||
or :meth:`from_crawler` method to indicate that it will remain disabled.
|
|
||||||
|
|
||||||
Only the following components can be disabled this way:
|
|
||||||
|
|
||||||
- :ref:`Downloader middlewares <topics-downloader-middleware>`
|
|
||||||
- :ref:`Extensions <topics-extensions>`
|
|
||||||
- :ref:`Item pipelines <topics-item-pipeline>`
|
|
||||||
- :ref:`Spider middlewares <topics-spider-middleware>`"""
|
|
||||||
|
|
||||||
|
|
||||||
class _InvalidOutput(TypeError):
|
class _InvalidOutput(TypeError):
|
||||||
|
|
@ -38,37 +30,15 @@ class _InvalidOutput(TypeError):
|
||||||
|
|
||||||
|
|
||||||
class IgnoreRequest(Exception):
|
class IgnoreRequest(Exception):
|
||||||
"""Raised to indicate that a request should be ignored.
|
"""Indicates a decision was made not to process a request"""
|
||||||
|
|
||||||
A :ref:`downloader middleware <topics-downloader-middleware>` can raise it
|
|
||||||
from its
|
|
||||||
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`
|
|
||||||
or
|
|
||||||
:meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`
|
|
||||||
method to drop a request, and a :signal:`request_scheduled` signal handler
|
|
||||||
can raise it to drop a request before it reaches the
|
|
||||||
:ref:`scheduler <topics-scheduler>`."""
|
|
||||||
|
|
||||||
|
|
||||||
class DontCloseSpider(Exception):
|
class DontCloseSpider(Exception):
|
||||||
"""Raised in a :signal:`spider_idle` signal handler to prevent the spider
|
"""Request the spider not to be closed yet"""
|
||||||
from being closed."""
|
|
||||||
|
|
||||||
|
|
||||||
class CloseSpider(Exception):
|
class CloseSpider(Exception):
|
||||||
"""Raised from a :ref:`spider callback <topics-spiders>` to request the
|
"""Raise this from callbacks to request the spider to be closed"""
|
||||||
spider to be closed/stopped.
|
|
||||||
|
|
||||||
*reason* is a string with the reason for closing.
|
|
||||||
|
|
||||||
For example:
|
|
||||||
|
|
||||||
.. code-block:: python
|
|
||||||
|
|
||||||
def parse_page(self, response):
|
|
||||||
if "Bandwidth exceeded" in response.text:
|
|
||||||
raise CloseSpider("bandwidth_exceeded")
|
|
||||||
"""
|
|
||||||
|
|
||||||
def __init__(self, reason: str = "cancelled"):
|
def __init__(self, reason: str = "cancelled"):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
@ -76,27 +46,10 @@ class CloseSpider(Exception):
|
||||||
|
|
||||||
|
|
||||||
class StopDownload(Exception):
|
class StopDownload(Exception):
|
||||||
"""Raised from a :class:`~scrapy.signals.bytes_received` or
|
"""
|
||||||
:class:`~scrapy.signals.headers_received` signal handler to :ref:`stop the
|
Stop the download of the body for a given response.
|
||||||
download <topics-stop-response-download>` of the response body.
|
The 'fail' boolean parameter indicates whether or not the resulting partial response
|
||||||
|
should be handled by the request errback. Note that 'fail' is a keyword-only argument.
|
||||||
The ``fail`` boolean parameter controls which method will handle the
|
|
||||||
resulting response:
|
|
||||||
|
|
||||||
* If ``fail=True`` (default), the request errback is called. The response
|
|
||||||
object is available as the ``response`` attribute of the ``StopDownload``
|
|
||||||
exception, which is in turn stored as the ``value`` attribute of the
|
|
||||||
received :class:`~twisted.python.failure.Failure` object. This means that
|
|
||||||
in an errback defined as ``def errback(self, failure)``, the response can
|
|
||||||
be accessed though ``failure.value.response``.
|
|
||||||
|
|
||||||
* If ``fail=False``, the request callback is called instead.
|
|
||||||
|
|
||||||
In both cases, the response could have its body truncated: the body contains
|
|
||||||
all bytes received up until the exception is raised, including the bytes
|
|
||||||
received in the signal handler that raises the exception. Also, the response
|
|
||||||
object is marked with ``"download_stopped"`` in its
|
|
||||||
:attr:`~scrapy.http.Response.flags` attribute.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
response: Response | None
|
response: Response | None
|
||||||
|
|
@ -138,8 +91,7 @@ class UnsupportedURLSchemeError(Exception):
|
||||||
|
|
||||||
|
|
||||||
class DropItem(Exception):
|
class DropItem(Exception):
|
||||||
"""Raised from the :meth:`process_item` method of an :ref:`item pipeline
|
"""Drop item from the item pipeline"""
|
||||||
<topics-item-pipeline>` to stop the processing of an item."""
|
|
||||||
|
|
||||||
def __init__(self, message: str, log_level: str | None = None):
|
def __init__(self, message: str, log_level: str | None = None):
|
||||||
super().__init__(message)
|
super().__init__(message)
|
||||||
|
|
@ -147,14 +99,7 @@ class DropItem(Exception):
|
||||||
|
|
||||||
|
|
||||||
class NotSupported(Exception):
|
class NotSupported(Exception):
|
||||||
"""Raised to indicate that a requested feature is not supported.
|
"""Indicates a feature or method is not supported"""
|
||||||
|
|
||||||
For example, Scrapy raises it when text-parsing shortcuts such as
|
|
||||||
:meth:`response.css() <scrapy.http.TextResponse.css>` or
|
|
||||||
:meth:`response.xpath() <scrapy.http.TextResponse.xpath>` are used on a
|
|
||||||
:class:`~scrapy.http.Response` whose content is not text, or when sending a
|
|
||||||
request whose URL scheme has no matching :ref:`download handler
|
|
||||||
<topics-download-handlers>`."""
|
|
||||||
|
|
||||||
|
|
||||||
# Commands
|
# Commands
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,6 @@ Item Exporters are used to export/serialize items into different formats.
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import csv
|
import csv
|
||||||
import logging
|
|
||||||
import marshal
|
import marshal
|
||||||
import pickle
|
import pickle
|
||||||
import pprint
|
import pprint
|
||||||
|
|
@ -25,8 +24,6 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from json import JSONEncoder
|
from json import JSONEncoder
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"BaseItemExporter",
|
"BaseItemExporter",
|
||||||
"CsvItemExporter",
|
"CsvItemExporter",
|
||||||
|
|
@ -74,17 +71,6 @@ class BaseItemExporter(ABC):
|
||||||
def finish_exporting(self) -> None: # noqa: B027
|
def finish_exporting(self) -> None: # noqa: B027
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def _get_populated_field_names(adapter: ItemAdapter) -> Iterable[str]:
|
|
||||||
"""Return the populated field names of *adapter*, in declaration order.
|
|
||||||
|
|
||||||
Populated fields that are not declared, which some item types allow,
|
|
||||||
come last, in item order.
|
|
||||||
"""
|
|
||||||
populated = set(adapter.keys())
|
|
||||||
declared = (name for name in adapter.field_names() if name in populated)
|
|
||||||
return dict.fromkeys([*declared, *adapter.keys()])
|
|
||||||
|
|
||||||
def _get_serialized_fields(
|
def _get_serialized_fields(
|
||||||
self, item: Any, default_value: Any = None, include_empty: bool | None = None
|
self, item: Any, default_value: Any = None, include_empty: bool | None = None
|
||||||
) -> Iterable[tuple[str, Any]]:
|
) -> Iterable[tuple[str, Any]]:
|
||||||
|
|
@ -97,11 +83,7 @@ class BaseItemExporter(ABC):
|
||||||
include_empty = self.export_empty_fields
|
include_empty = self.export_empty_fields
|
||||||
|
|
||||||
if self.fields_to_export is None:
|
if self.fields_to_export is None:
|
||||||
field_iter = (
|
field_iter = item.field_names() if include_empty else item.keys()
|
||||||
item.field_names()
|
|
||||||
if include_empty
|
|
||||||
else self._get_populated_field_names(item)
|
|
||||||
)
|
|
||||||
elif isinstance(self.fields_to_export, Mapping):
|
elif isinstance(self.fields_to_export, Mapping):
|
||||||
if include_empty:
|
if include_empty:
|
||||||
field_iter = self.fields_to_export.items()
|
field_iter = self.fields_to_export.items()
|
||||||
|
|
@ -272,8 +254,6 @@ class CsvItemExporter(BaseItemExporter):
|
||||||
self.csv_writer = csv.writer(self.stream, **self._kwargs)
|
self.csv_writer = csv.writer(self.stream, **self._kwargs)
|
||||||
self._headers_not_written = True
|
self._headers_not_written = True
|
||||||
self._join_multivalued = join_multivalued
|
self._join_multivalued = join_multivalued
|
||||||
self._autodetected_fields = False
|
|
||||||
self._data_loss_warned = False
|
|
||||||
|
|
||||||
def serialize_field(
|
def serialize_field(
|
||||||
self, field: Mapping[str, Any] | Field, name: str, value: Any
|
self, field: Mapping[str, Any] | Field, name: str, value: Any
|
||||||
|
|
@ -294,22 +274,6 @@ class CsvItemExporter(BaseItemExporter):
|
||||||
self._headers_not_written = False
|
self._headers_not_written = False
|
||||||
self._write_headers_and_set_fields_to_export(item)
|
self._write_headers_and_set_fields_to_export(item)
|
||||||
|
|
||||||
if (
|
|
||||||
self._autodetected_fields
|
|
||||||
and self.fields_to_export is not None
|
|
||||||
and not self._data_loss_warned
|
|
||||||
):
|
|
||||||
item_fields = ItemAdapter(item).field_names()
|
|
||||||
dropped_fields = set(item_fields) - set(self.fields_to_export)
|
|
||||||
|
|
||||||
if dropped_fields:
|
|
||||||
dropped_fields_display = sorted(dropped_fields)
|
|
||||||
logger.warning(
|
|
||||||
f"CSVExporter dropped fields {dropped_fields_display}. "
|
|
||||||
f"To avoid this, fully configure your FEED_EXPORT_FIELDS setting. "
|
|
||||||
f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields",
|
|
||||||
)
|
|
||||||
self._data_loss_warned = True
|
|
||||||
fields = self._get_serialized_fields(item, default_value="", include_empty=True)
|
fields = self._get_serialized_fields(item, default_value="", include_empty=True)
|
||||||
values = list(self._build_row(x for _, x in fields))
|
values = list(self._build_row(x for _, x in fields))
|
||||||
self.csv_writer.writerow(values)
|
self.csv_writer.writerow(values)
|
||||||
|
|
@ -329,7 +293,6 @@ class CsvItemExporter(BaseItemExporter):
|
||||||
if not self.fields_to_export:
|
if not self.fields_to_export:
|
||||||
# use declared field names, or keys if the item is a dict
|
# use declared field names, or keys if the item is a dict
|
||||||
self.fields_to_export = ItemAdapter(item).field_names()
|
self.fields_to_export = ItemAdapter(item).field_names()
|
||||||
self._autodetected_fields = True
|
|
||||||
fields: Iterable[str]
|
fields: Iterable[str]
|
||||||
if isinstance(self.fields_to_export, Mapping):
|
if isinstance(self.fields_to_export, Mapping):
|
||||||
fields = self.fields_to_export.values()
|
fields = self.fields_to_export.values()
|
||||||
|
|
|
||||||
|
|
@ -119,7 +119,7 @@ class CloseSpider:
|
||||||
self.task = None
|
self.task = None
|
||||||
|
|
||||||
if self.task_no_item:
|
if self.task_no_item:
|
||||||
if self.task_no_item.running: # pragma: no branch
|
if self.task_no_item.running:
|
||||||
self.task_no_item.stop()
|
self.task_no_item.stop()
|
||||||
self.task_no_item = None
|
self.task_no_item = None
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ import re
|
||||||
import sys
|
import sys
|
||||||
import warnings
|
import warnings
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable, Coroutine
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path, PureWindowsPath
|
from pathlib import Path, PureWindowsPath
|
||||||
from tempfile import NamedTemporaryFile
|
from tempfile import NamedTemporaryFile
|
||||||
|
|
@ -22,13 +22,12 @@ from urllib.parse import unquote, urlparse
|
||||||
|
|
||||||
from twisted.internet.defer import Deferred, DeferredList
|
from twisted.internet.defer import Deferred, DeferredList
|
||||||
from w3lib.url import file_uri_to_path
|
from w3lib.url import file_uri_to_path
|
||||||
from zope.interface import Interface
|
from zope.interface import Interface, implementer
|
||||||
|
|
||||||
from scrapy import Spider, signals
|
from scrapy import Spider, signals
|
||||||
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
|
||||||
from scrapy.extensions.postprocessing import PostProcessingManager
|
from scrapy.extensions.postprocessing import PostProcessingManager
|
||||||
from scrapy.utils.asyncio import is_asyncio_available, run_in_thread
|
from scrapy.utils.asyncio import is_asyncio_available, run_in_thread
|
||||||
from scrapy.utils.boto import _get_max_pool_connections
|
|
||||||
from scrapy.utils.conf import feed_complete_default_values_from_settings
|
from scrapy.utils.conf import feed_complete_default_values_from_settings
|
||||||
from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
|
from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
|
||||||
from scrapy.utils.ftp import ftp_store_file
|
from scrapy.utils.ftp import ftp_store_file
|
||||||
|
|
@ -48,33 +47,6 @@ if TYPE_CHECKING:
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
# Printf-style placeholders (e.g. %(time)s) used to build feed URIs. Any other
|
|
||||||
# percent character in a URI (e.g. percent-encoding such as %20 or %23) must be
|
|
||||||
# treated as a literal rather than as the start of a placeholder.
|
|
||||||
_FEED_URI_PLACEHOLDER_RE = re.compile(
|
|
||||||
r"%\([^)]+\)[-+ #0]*(?:\d+|\*)?(?:\.(?:\d+|\*))?[diouxXeEfFgGcrsa]"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def apply_uri_params(uri_template: str, uri_params: dict[str, Any]) -> str:
|
|
||||||
"""Return *uri_template* with its ``%(...)s`` placeholders replaced using
|
|
||||||
*uri_params*, leaving any other percent character untouched.
|
|
||||||
|
|
||||||
This allows feed URIs to contain percent-encoded characters (e.g. ``%20``
|
|
||||||
in a path with spaces or ``%23`` in FTP credentials) without them being
|
|
||||||
misinterpreted as printf-style formatting directives.
|
|
||||||
"""
|
|
||||||
parts: list[str] = []
|
|
||||||
last = 0
|
|
||||||
for match in _FEED_URI_PLACEHOLDER_RE.finditer(uri_template):
|
|
||||||
parts.append(uri_template[last : match.start()].replace("%", "%%"))
|
|
||||||
parts.append(match.group(0))
|
|
||||||
last = match.end()
|
|
||||||
parts.append(uri_template[last:].replace("%", "%%"))
|
|
||||||
return "".join(parts) % uri_params
|
|
||||||
|
|
||||||
|
|
||||||
UriParamsCallableT: TypeAlias = Callable[
|
UriParamsCallableT: TypeAlias = Callable[
|
||||||
[dict[str, Any], Spider], dict[str, Any] | None
|
[dict[str, Any], Spider], dict[str, Any] | None
|
||||||
]
|
]
|
||||||
|
|
@ -116,18 +88,25 @@ class ItemFilter:
|
||||||
return True # accept all items by default
|
return True # accept all items by default
|
||||||
|
|
||||||
|
|
||||||
class _IFeedStorage(Interface): # type: ignore[misc] # pragma: no cover
|
class IFeedStorage(Interface): # type: ignore[misc]
|
||||||
|
"""Interface that all Feed Storages must implement"""
|
||||||
|
|
||||||
# pylint: disable=no-self-argument
|
# pylint: disable=no-self-argument
|
||||||
|
|
||||||
def __init__(uri, *, feed_options=None): ... # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called
|
def __init__(uri, *, feed_options=None): # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called
|
||||||
|
"""Initialize the storage with the parameters given in the URI and the
|
||||||
|
feed-specific options (see :setting:`FEEDS`)"""
|
||||||
|
|
||||||
def open(spider): ... # type: ignore[no-untyped-def]
|
def open(spider): # type: ignore[no-untyped-def]
|
||||||
|
"""Open the storage for the given spider. It must return a file-like
|
||||||
|
object that will be used for the exporters"""
|
||||||
|
|
||||||
def store(file): ... # type: ignore[no-untyped-def]
|
def store(file): # type: ignore[no-untyped-def]
|
||||||
|
"""Store the given file stream"""
|
||||||
|
|
||||||
|
|
||||||
class FeedStorageProtocol(Protocol):
|
class FeedStorageProtocol(Protocol):
|
||||||
"""Protocol that all Feed Storages must follow."""
|
"""Reimplementation of ``IFeedStorage`` that can be used in type hints."""
|
||||||
|
|
||||||
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
|
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
|
||||||
"""Initialize the storage with the parameters given in the URI and the
|
"""Initialize the storage with the parameters given in the URI and the
|
||||||
|
|
@ -141,6 +120,7 @@ class FeedStorageProtocol(Protocol):
|
||||||
"""Store the given file stream"""
|
"""Store the given file stream"""
|
||||||
|
|
||||||
|
|
||||||
|
@implementer(IFeedStorage)
|
||||||
class BlockingFeedStorage(ABC):
|
class BlockingFeedStorage(ABC):
|
||||||
def open(self, spider: Spider) -> IO[bytes]:
|
def open(self, spider: Spider) -> IO[bytes]:
|
||||||
path = spider.crawler.settings["FEED_TEMPDIR"]
|
path = spider.crawler.settings["FEED_TEMPDIR"]
|
||||||
|
|
@ -157,6 +137,7 @@ class BlockingFeedStorage(ABC):
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
|
||||||
|
@implementer(IFeedStorage)
|
||||||
class StdoutFeedStorage:
|
class StdoutFeedStorage:
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
|
|
@ -183,6 +164,7 @@ class StdoutFeedStorage:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@implementer(IFeedStorage)
|
||||||
class FileFeedStorage:
|
class FileFeedStorage:
|
||||||
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
|
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
|
||||||
self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri
|
self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri
|
||||||
|
|
@ -214,14 +196,11 @@ class S3FeedStorage(BlockingFeedStorage):
|
||||||
feed_options: dict[str, Any] | None = None,
|
feed_options: dict[str, Any] | None = None,
|
||||||
session_token: str | None = None,
|
session_token: str | None = None,
|
||||||
region_name: str | None = None,
|
region_name: str | None = None,
|
||||||
max_pool_connections: int | None = None,
|
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
import boto3.session # noqa: PLC0415
|
import boto3.session # noqa: PLC0415
|
||||||
except ImportError:
|
except ImportError:
|
||||||
raise NotConfigured("missing boto3 library") from None
|
raise NotConfigured("missing boto3 library") from None
|
||||||
from botocore.config import Config # noqa: PLC0415
|
|
||||||
|
|
||||||
u = urlparse(uri)
|
u = urlparse(uri)
|
||||||
assert u.hostname
|
assert u.hostname
|
||||||
self.bucketname: str = u.hostname
|
self.bucketname: str = u.hostname
|
||||||
|
|
@ -232,7 +211,6 @@ class S3FeedStorage(BlockingFeedStorage):
|
||||||
self.acl: str | None = acl
|
self.acl: str | None = acl
|
||||||
self.endpoint_url: str | None = endpoint_url
|
self.endpoint_url: str | None = endpoint_url
|
||||||
self.region_name: str | None = region_name
|
self.region_name: str | None = region_name
|
||||||
self.max_pool_connections: int | None = max_pool_connections
|
|
||||||
|
|
||||||
boto3_session = boto3.session.Session()
|
boto3_session = boto3.session.Session()
|
||||||
self.s3_client = boto3_session.client(
|
self.s3_client = boto3_session.client(
|
||||||
|
|
@ -242,11 +220,6 @@ class S3FeedStorage(BlockingFeedStorage):
|
||||||
aws_session_token=self.session_token,
|
aws_session_token=self.session_token,
|
||||||
endpoint_url=self.endpoint_url,
|
endpoint_url=self.endpoint_url,
|
||||||
region_name=self.region_name,
|
region_name=self.region_name,
|
||||||
config=(
|
|
||||||
Config(max_pool_connections=self.max_pool_connections)
|
|
||||||
if self.max_pool_connections is not None
|
|
||||||
else None
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if feed_options and feed_options.get("overwrite", True) is False:
|
if feed_options and feed_options.get("overwrite", True) is False:
|
||||||
|
|
@ -272,7 +245,6 @@ class S3FeedStorage(BlockingFeedStorage):
|
||||||
acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None,
|
acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None,
|
||||||
endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,
|
endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,
|
||||||
region_name=crawler.settings["AWS_REGION_NAME"] or None,
|
region_name=crawler.settings["AWS_REGION_NAME"] or None,
|
||||||
max_pool_connections=_get_max_pool_connections(crawler.settings),
|
|
||||||
feed_options=feed_options,
|
feed_options=feed_options,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -465,7 +437,7 @@ class FeedSlot:
|
||||||
)
|
)
|
||||||
|
|
||||||
def finish_exporting(self) -> None:
|
def finish_exporting(self) -> None:
|
||||||
if self._exporting: # pragma: no branch
|
if self._exporting:
|
||||||
assert self.exporter
|
assert self.exporter
|
||||||
self.exporter.finish_exporting()
|
self.exporter.finish_exporting()
|
||||||
self._exporting = False
|
self._exporting = False
|
||||||
|
|
@ -486,7 +458,7 @@ class FeedExporter:
|
||||||
self.feeds = {}
|
self.feeds = {}
|
||||||
self.slots: list[FeedSlot] = []
|
self.slots: list[FeedSlot] = []
|
||||||
self.filters: dict[str, ItemFilter] = {}
|
self.filters: dict[str, ItemFilter] = {}
|
||||||
self._pending_close_tasks: list[asyncio.Task[None] | Deferred[None]] = []
|
self._pending_close_coros: list[Coroutine[Any, Any, None]] = []
|
||||||
|
|
||||||
if not self.settings["FEEDS"] and not self.settings["FEED_URI"]:
|
if not self.settings["FEEDS"] and not self.settings["FEED_URI"]:
|
||||||
raise NotConfigured
|
raise NotConfigured
|
||||||
|
|
@ -501,7 +473,7 @@ class FeedExporter:
|
||||||
)
|
)
|
||||||
uri = self.settings["FEED_URI"]
|
uri = self.settings["FEED_URI"]
|
||||||
# handle pathlib.Path objects
|
# handle pathlib.Path objects
|
||||||
uri = str(uri.absolute()) if isinstance(uri, Path) else str(uri)
|
uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
|
||||||
feed_options = {"format": self.settings["FEED_FORMAT"]}
|
feed_options = {"format": self.settings["FEED_FORMAT"]}
|
||||||
self.feeds[uri] = feed_complete_default_values_from_settings(
|
self.feeds[uri] = feed_complete_default_values_from_settings(
|
||||||
feed_options, self.settings
|
feed_options, self.settings
|
||||||
|
|
@ -513,9 +485,9 @@ class FeedExporter:
|
||||||
for settings_uri, feed_options in self.settings.getdict("FEEDS").items():
|
for settings_uri, feed_options in self.settings.getdict("FEEDS").items():
|
||||||
# handle pathlib.Path objects
|
# handle pathlib.Path objects
|
||||||
uri = (
|
uri = (
|
||||||
str(settings_uri.absolute())
|
str(settings_uri)
|
||||||
if isinstance(settings_uri, Path)
|
if not isinstance(settings_uri, Path)
|
||||||
else str(settings_uri)
|
else settings_uri.absolute().as_uri()
|
||||||
)
|
)
|
||||||
self.feeds[uri] = feed_complete_default_values_from_settings(
|
self.feeds[uri] = feed_complete_default_values_from_settings(
|
||||||
feed_options, self.settings
|
feed_options, self.settings
|
||||||
|
|
@ -542,7 +514,7 @@ class FeedExporter:
|
||||||
self.slots.append(
|
self.slots.append(
|
||||||
self._start_new_batch(
|
self._start_new_batch(
|
||||||
batch_id=1,
|
batch_id=1,
|
||||||
uri=apply_uri_params(uri, uri_params),
|
uri=uri % uri_params,
|
||||||
feed_options=feed_options,
|
feed_options=feed_options,
|
||||||
spider=spider,
|
spider=spider,
|
||||||
uri_template=uri,
|
uri_template=uri,
|
||||||
|
|
@ -550,44 +522,23 @@ class FeedExporter:
|
||||||
)
|
)
|
||||||
|
|
||||||
async def close_spider(self, spider: Spider) -> None:
|
async def close_spider(self, spider: Spider) -> None:
|
||||||
for slot in self.slots:
|
self._pending_close_coros.extend(
|
||||||
self._schedule_slot_close(slot, spider)
|
self._close_slot(slot, spider) for slot in self.slots
|
||||||
|
)
|
||||||
|
|
||||||
if self._pending_close_tasks: # pragma: no branch
|
if self._pending_close_coros:
|
||||||
if is_asyncio_available():
|
if is_asyncio_available():
|
||||||
await asyncio.wait(
|
await asyncio.wait(
|
||||||
cast("list[asyncio.Task[None]]", list(self._pending_close_tasks))
|
[asyncio.create_task(coro) for coro in self._pending_close_coros]
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
await DeferredList(
|
await DeferredList(
|
||||||
cast("list[Deferred[None]]", list(self._pending_close_tasks))
|
deferred_from_coro(coro) for coro in self._pending_close_coros
|
||||||
)
|
)
|
||||||
|
|
||||||
# Send FEED_EXPORTER_CLOSED signal
|
# Send FEED_EXPORTER_CLOSED signal
|
||||||
await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed)
|
await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed)
|
||||||
|
|
||||||
def _schedule_slot_close(
|
|
||||||
self, slot: FeedSlot, spider: Spider
|
|
||||||
) -> asyncio.Task[None] | Deferred[None]:
|
|
||||||
"""Start closing the slot without waiting for it to finish, keeping
|
|
||||||
track of the pending work so that it can be awaited in
|
|
||||||
:meth:`close_spider` if it hasn't finished by then."""
|
|
||||||
aw: asyncio.Task[None] | Deferred[None]
|
|
||||||
coro = self._close_slot(slot, spider)
|
|
||||||
if is_asyncio_available():
|
|
||||||
aw = asyncio.create_task(coro)
|
|
||||||
self._pending_close_tasks.append(aw)
|
|
||||||
aw.add_done_callback(self._pending_close_tasks.remove)
|
|
||||||
else:
|
|
||||||
aw = deferred_from_coro(coro)
|
|
||||||
self._pending_close_tasks.append(aw)
|
|
||||||
aw.addBoth(self._untrack_pending_close_task, aw)
|
|
||||||
return aw
|
|
||||||
|
|
||||||
def _untrack_pending_close_task(self, result: Any, aw: Deferred[None]) -> Any:
|
|
||||||
self._pending_close_tasks.remove(aw)
|
|
||||||
return result
|
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _get_file(slot_: FeedSlot) -> IO[bytes]:
|
def _get_file(slot_: FeedSlot) -> IO[bytes]:
|
||||||
assert slot_.file
|
assert slot_.file
|
||||||
|
|
@ -684,11 +635,11 @@ class FeedExporter:
|
||||||
uri_params = self._get_uri_params(
|
uri_params = self._get_uri_params(
|
||||||
spider, self.feeds[slot.uri_template]["uri_params"], slot
|
spider, self.feeds[slot.uri_template]["uri_params"], slot
|
||||||
)
|
)
|
||||||
self._schedule_slot_close(slot, spider)
|
self._pending_close_coros.append(self._close_slot(slot, spider))
|
||||||
slots.append(
|
slots.append(
|
||||||
self._start_new_batch(
|
self._start_new_batch(
|
||||||
batch_id=slot.batch_id + 1,
|
batch_id=slot.batch_id + 1,
|
||||||
uri=apply_uri_params(slot.uri_template, uri_params),
|
uri=slot.uri_template % uri_params,
|
||||||
feed_options=self.feeds[slot.uri_template],
|
feed_options=self.feeds[slot.uri_template],
|
||||||
spider=spider,
|
spider=spider,
|
||||||
uri_template=slot.uri_template,
|
uri_template=slot.uri_template,
|
||||||
|
|
@ -780,14 +731,3 @@ class FeedExporter:
|
||||||
feed_options.get("item_filter", ItemFilter)
|
feed_options.get("item_filter", ItemFilter)
|
||||||
)
|
)
|
||||||
return item_filter_class(feed_options)
|
return item_filter_class(feed_options)
|
||||||
|
|
||||||
|
|
||||||
def __getattr__(name: str) -> Any: # pragma: no cover
|
|
||||||
if name == "IFeedStorage":
|
|
||||||
warnings.warn(
|
|
||||||
"scrapy.extensions.feedexport.IFeedStorage is deprecated.",
|
|
||||||
ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
return _IFeedStorage
|
|
||||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ class LogCount:
|
||||||
"""Install a log handler that counts log messages by level.
|
"""Install a log handler that counts log messages by level.
|
||||||
|
|
||||||
The handler installed is :class:`scrapy.utils.log.LogCounterHandler`.
|
The handler installed is :class:`scrapy.utils.log.LogCounterHandler`.
|
||||||
The counts are stored in the :stat:`log_count/{level}` stat.
|
The counts are stored in stats as ``log_count/<level>``.
|
||||||
|
|
||||||
.. versionadded:: 2.14
|
.. versionadded:: 2.14
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -94,7 +94,7 @@ class MemoryUsage:
|
||||||
|
|
||||||
def engine_stopped(self) -> None:
|
def engine_stopped(self) -> None:
|
||||||
for tsk in self.tasks:
|
for tsk in self.tasks:
|
||||||
if tsk.running: # pragma: no branch
|
if tsk.running:
|
||||||
tsk.stop()
|
tsk.stop()
|
||||||
|
|
||||||
def update(self) -> None:
|
def update(self) -> None:
|
||||||
|
|
|
||||||
|
|
@ -38,6 +38,7 @@ class PeriodicLog:
|
||||||
):
|
):
|
||||||
self.stats: StatsCollector = stats
|
self.stats: StatsCollector = stats
|
||||||
self.interval: float = interval
|
self.interval: float = interval
|
||||||
|
self.multiplier: float = 60.0 / self.interval
|
||||||
self.task: AsyncioLoopingCall | LoopingCall | None = None
|
self.task: AsyncioLoopingCall | LoopingCall | None = None
|
||||||
self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
|
self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
|
||||||
self.ext_stats_enabled: bool = bool(ext_stats)
|
self.ext_stats_enabled: bool = bool(ext_stats)
|
||||||
|
|
@ -164,5 +165,5 @@ class PeriodicLog:
|
||||||
|
|
||||||
def spider_closed(self, spider: Spider, reason: str) -> None:
|
def spider_closed(self, spider: Spider, reason: str) -> None:
|
||||||
self.log()
|
self.log()
|
||||||
if self.task and self.task.running: # pragma: no branch
|
if self.task and self.task.running:
|
||||||
self.task.stop()
|
self.task.stop()
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from typing import TYPE_CHECKING, Any, TypeAlias, cast
|
from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast
|
||||||
|
|
||||||
from w3lib.http import headers_dict_to_raw
|
from w3lib.http import headers_dict_to_raw
|
||||||
|
|
||||||
|
|
@ -25,20 +25,14 @@ class Headers(CaselessDict):
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
seq: Mapping[str, Any]
|
seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
|
||||||
| Mapping[bytes, Any]
|
|
||||||
| Iterable[tuple[str | bytes, Any]]
|
|
||||||
| None = None,
|
|
||||||
encoding: str = "utf-8",
|
encoding: str = "utf-8",
|
||||||
):
|
):
|
||||||
self.encoding: str = encoding
|
self.encoding: str = encoding
|
||||||
super().__init__(seq)
|
super().__init__(seq)
|
||||||
|
|
||||||
def update( # type: ignore[override]
|
def update( # type: ignore[override]
|
||||||
self,
|
self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]
|
||||||
seq: Mapping[str, Any]
|
|
||||||
| Mapping[bytes, Any]
|
|
||||||
| Iterable[tuple[str | bytes, Any]],
|
|
||||||
) -> None:
|
) -> None:
|
||||||
seq = seq.items() if isinstance(seq, Mapping) else seq
|
seq = seq.items() if isinstance(seq, Mapping) else seq
|
||||||
iseq: dict[bytes, list[bytes]] = {}
|
iseq: dict[bytes, list[bytes]] = {}
|
||||||
|
|
@ -46,7 +40,7 @@ class Headers(CaselessDict):
|
||||||
iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
|
iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
|
||||||
super().update(iseq)
|
super().update(iseq)
|
||||||
|
|
||||||
def normkey(self, key: str | bytes) -> bytes:
|
def normkey(self, key: AnyStr) -> bytes: # type: ignore[override]
|
||||||
"""Normalize key to bytes"""
|
"""Normalize key to bytes"""
|
||||||
return self._tobytes(key.title())
|
return self._tobytes(key.title())
|
||||||
|
|
||||||
|
|
@ -73,19 +67,19 @@ class Headers(CaselessDict):
|
||||||
return str(x).encode(self.encoding)
|
return str(x).encode(self.encoding)
|
||||||
raise TypeError(f"Unsupported value type: {type(x)}")
|
raise TypeError(f"Unsupported value type: {type(x)}")
|
||||||
|
|
||||||
def __getitem__(self, key: str | bytes) -> bytes | None:
|
def __getitem__(self, key: AnyStr) -> bytes | None:
|
||||||
try:
|
try:
|
||||||
return cast("list[bytes]", super().__getitem__(key))[-1]
|
return cast("list[bytes]", super().__getitem__(key))[-1]
|
||||||
except IndexError:
|
except IndexError:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def get(self, key: str | bytes, def_val: Any = None) -> bytes | None:
|
def get(self, key: AnyStr, def_val: Any = None) -> bytes | None:
|
||||||
try:
|
try:
|
||||||
return cast("list[bytes]", super().get(key, def_val))[-1]
|
return cast("list[bytes]", super().get(key, def_val))[-1]
|
||||||
except IndexError:
|
except IndexError:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def getlist(self, key: str | bytes, def_val: Any = None) -> list[bytes]:
|
def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]:
|
||||||
try:
|
try:
|
||||||
return cast("list[bytes]", super().__getitem__(key))
|
return cast("list[bytes]", super().__getitem__(key))
|
||||||
except KeyError:
|
except KeyError:
|
||||||
|
|
@ -93,15 +87,15 @@ class Headers(CaselessDict):
|
||||||
return self.normvalue(def_val)
|
return self.normvalue(def_val)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
def setlist(self, key: str | bytes, list_: Iterable[_RawValue]) -> None:
|
def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None:
|
||||||
self[key] = list_
|
self[key] = list_
|
||||||
|
|
||||||
def setlistdefault(
|
def setlistdefault(
|
||||||
self, key: str | bytes, default_list: Iterable[_RawValue] = ()
|
self, key: AnyStr, default_list: Iterable[_RawValue] = ()
|
||||||
) -> Any:
|
) -> Any:
|
||||||
return self.setdefault(key, default_list)
|
return self.setdefault(key, default_list)
|
||||||
|
|
||||||
def appendlist(self, key: str | bytes, value: Iterable[_RawValue]) -> None:
|
def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None:
|
||||||
lst = self.getlist(key)
|
lst = self.getlist(key)
|
||||||
lst.extend(self.normvalue(value))
|
lst.extend(self.normvalue(value))
|
||||||
self[key] = lst
|
self[key] = lst
|
||||||
|
|
|
||||||
|
|
@ -11,6 +11,7 @@ import inspect
|
||||||
from typing import (
|
from typing import (
|
||||||
TYPE_CHECKING,
|
TYPE_CHECKING,
|
||||||
Any,
|
Any,
|
||||||
|
AnyStr,
|
||||||
Concatenate,
|
Concatenate,
|
||||||
NoReturn,
|
NoReturn,
|
||||||
TypeAlias,
|
TypeAlias,
|
||||||
|
|
@ -124,10 +125,7 @@ class Request(object_ref):
|
||||||
url: str,
|
url: str,
|
||||||
callback: CallbackT | None = None,
|
callback: CallbackT | None = None,
|
||||||
method: str = "GET",
|
method: str = "GET",
|
||||||
headers: Mapping[str, Any]
|
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
|
||||||
| Mapping[bytes, Any]
|
|
||||||
| Iterable[tuple[str | bytes, Any]]
|
|
||||||
| None = None,
|
|
||||||
body: bytes | str | None = None,
|
body: bytes | str | None = None,
|
||||||
cookies: CookiesT | None = None,
|
cookies: CookiesT | None = None,
|
||||||
meta: dict[str, Any] | None = None,
|
meta: dict[str, Any] | None = None,
|
||||||
|
|
@ -169,8 +167,7 @@ class Request(object_ref):
|
||||||
#:
|
#:
|
||||||
#: The callable must expect the response as its first parameter, and
|
#: The callable must expect the response as its first parameter, and
|
||||||
#: support any additional keyword arguments set through
|
#: support any additional keyword arguments set through
|
||||||
#: :attr:`cb_kwargs`. See :ref:`writing-callbacks` and
|
#: :attr:`cb_kwargs`.
|
||||||
#: :ref:`callback-output`.
|
|
||||||
#:
|
#:
|
||||||
#: In addition to an arbitrary callable, the following values are also
|
#: In addition to an arbitrary callable, the following values are also
|
||||||
#: supported:
|
#: supported:
|
||||||
|
|
@ -191,7 +188,8 @@ class Request(object_ref):
|
||||||
#: raises exceptions for non-2xx responses by default, sending them
|
#: raises exceptions for non-2xx responses by default, sending them
|
||||||
#: to the :attr:`errback` instead.
|
#: to the :attr:`errback` instead.
|
||||||
#:
|
#:
|
||||||
#: .. seealso:: :ref:`callbacks`
|
#: .. seealso::
|
||||||
|
#: :ref:`topics-request-response-ref-request-callback-arguments`
|
||||||
self.callback: CallbackT | None = callback
|
self.callback: CallbackT | None = callback
|
||||||
|
|
||||||
#: :class:`~collections.abc.Callable` to handle exceptions raised
|
#: :class:`~collections.abc.Callable` to handle exceptions raised
|
||||||
|
|
@ -200,7 +198,7 @@ class Request(object_ref):
|
||||||
#: The callable must expect a :exc:`~twisted.python.failure.Failure` as
|
#: The callable must expect a :exc:`~twisted.python.failure.Failure` as
|
||||||
#: its first parameter.
|
#: its first parameter.
|
||||||
#:
|
#:
|
||||||
#: .. seealso:: :ref:`errbacks`
|
#: .. seealso:: :ref:`topics-request-response-ref-errbacks`
|
||||||
self.errback: Callable[[Failure], Any] | None = errback
|
self.errback: Callable[[Failure], Any] | None = errback
|
||||||
|
|
||||||
self._cookies: CookiesT | None = cookies or None
|
self._cookies: CookiesT | None = cookies or None
|
||||||
|
|
@ -312,11 +310,7 @@ class Request(object_ref):
|
||||||
|
|
||||||
@headers.setter
|
@headers.setter
|
||||||
def headers(
|
def headers(
|
||||||
self,
|
self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
|
||||||
value: Mapping[str, Any]
|
|
||||||
| Mapping[bytes, Any]
|
|
||||||
| Iterable[tuple[str | bytes, Any]]
|
|
||||||
| None,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
if isinstance(value, Headers):
|
if isinstance(value, Headers):
|
||||||
self._headers = value
|
self._headers = value
|
||||||
|
|
@ -387,20 +381,6 @@ class Request(object_ref):
|
||||||
request_kwargs.update(kwargs)
|
request_kwargs.update(kwargs)
|
||||||
return cls(**request_kwargs)
|
return cls(**request_kwargs)
|
||||||
|
|
||||||
def to_curl(self) -> str:
|
|
||||||
"""Return a string with a `cURL <https://curl.se/>`_ command equivalent
|
|
||||||
to this request.
|
|
||||||
|
|
||||||
Inverse of :meth:`from_curl`. See also
|
|
||||||
:func:`scrapy.utils.request.request_to_curl`.
|
|
||||||
|
|
||||||
.. versionadded:: VERSION
|
|
||||||
"""
|
|
||||||
# Imported here to avoid a circular import.
|
|
||||||
from scrapy.utils.request import request_to_curl # noqa: PLC0415
|
|
||||||
|
|
||||||
return request_to_curl(self)
|
|
||||||
|
|
||||||
def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]:
|
def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]:
|
||||||
"""Return a dictionary containing the Request's data.
|
"""Return a dictionary containing the Request's data.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from typing import TYPE_CHECKING, Any, TypeVar, overload
|
from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
from scrapy.exceptions import NotSupported
|
from scrapy.exceptions import NotSupported
|
||||||
|
|
@ -72,10 +72,7 @@ class Response(object_ref):
|
||||||
self,
|
self,
|
||||||
url: str,
|
url: str,
|
||||||
status: int = 200,
|
status: int = 200,
|
||||||
headers: Mapping[str, Any]
|
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
|
||||||
| Mapping[bytes, Any]
|
|
||||||
| Iterable[tuple[str | bytes, Any]]
|
|
||||||
| None = None,
|
|
||||||
body: bytes = b"",
|
body: bytes = b"",
|
||||||
flags: list[str] | None = None,
|
flags: list[str] | None = None,
|
||||||
request: Request | None = None,
|
request: Request | None = None,
|
||||||
|
|
@ -148,11 +145,7 @@ class Response(object_ref):
|
||||||
|
|
||||||
@headers.setter
|
@headers.setter
|
||||||
def headers(
|
def headers(
|
||||||
self,
|
self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
|
||||||
value: Mapping[str, Any]
|
|
||||||
| Mapping[bytes, Any]
|
|
||||||
| Iterable[tuple[str | bytes, Any]]
|
|
||||||
| None,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
if isinstance(value, Headers):
|
if isinstance(value, Headers):
|
||||||
self._headers = value
|
self._headers = value
|
||||||
|
|
@ -229,10 +222,7 @@ class Response(object_ref):
|
||||||
url: str | Link,
|
url: str | Link,
|
||||||
callback: CallbackT | None = None,
|
callback: CallbackT | None = None,
|
||||||
method: str = "GET",
|
method: str = "GET",
|
||||||
headers: Mapping[str, Any]
|
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
|
||||||
| Mapping[bytes, Any]
|
|
||||||
| Iterable[tuple[str | bytes, Any]]
|
|
||||||
| None = None,
|
|
||||||
body: bytes | str | None = None,
|
body: bytes | str | None = None,
|
||||||
cookies: CookiesT | None = None,
|
cookies: CookiesT | None = None,
|
||||||
meta: dict[str, Any] | None = None,
|
meta: dict[str, Any] | None = None,
|
||||||
|
|
@ -282,10 +272,7 @@ class Response(object_ref):
|
||||||
urls: Iterable[str | Link],
|
urls: Iterable[str | Link],
|
||||||
callback: CallbackT | None = None,
|
callback: CallbackT | None = None,
|
||||||
method: str = "GET",
|
method: str = "GET",
|
||||||
headers: Mapping[str, Any]
|
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
|
||||||
| Mapping[bytes, Any]
|
|
||||||
| Iterable[tuple[str | bytes, Any]]
|
|
||||||
| None = None,
|
|
||||||
body: bytes | str | None = None,
|
body: bytes | str | None = None,
|
||||||
cookies: CookiesT | None = None,
|
cookies: CookiesT | None = None,
|
||||||
meta: dict[str, Any] | None = None,
|
meta: dict[str, Any] | None = None,
|
||||||
|
|
|
||||||
|
|
@ -9,7 +9,7 @@ from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
from contextlib import suppress
|
from contextlib import suppress
|
||||||
from typing import TYPE_CHECKING, Any, cast
|
from typing import TYPE_CHECKING, Any, AnyStr, cast
|
||||||
from urllib.parse import urljoin
|
from urllib.parse import urljoin
|
||||||
|
|
||||||
import parsel
|
import parsel
|
||||||
|
|
@ -170,10 +170,7 @@ class TextResponse(Response):
|
||||||
url: str | Link | parsel.Selector,
|
url: str | Link | parsel.Selector,
|
||||||
callback: CallbackT | None = None,
|
callback: CallbackT | None = None,
|
||||||
method: str = "GET",
|
method: str = "GET",
|
||||||
headers: Mapping[str, Any]
|
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
|
||||||
| Mapping[bytes, Any]
|
|
||||||
| Iterable[tuple[str | bytes, Any]]
|
|
||||||
| None = None,
|
|
||||||
body: bytes | str | None = None,
|
body: bytes | str | None = None,
|
||||||
cookies: CookiesT | None = None,
|
cookies: CookiesT | None = None,
|
||||||
meta: dict[str, Any] | None = None,
|
meta: dict[str, Any] | None = None,
|
||||||
|
|
@ -226,10 +223,7 @@ class TextResponse(Response):
|
||||||
urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,
|
urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,
|
||||||
callback: CallbackT | None = None,
|
callback: CallbackT | None = None,
|
||||||
method: str = "GET",
|
method: str = "GET",
|
||||||
headers: Mapping[str, Any]
|
headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
|
||||||
| Mapping[bytes, Any]
|
|
||||||
| Iterable[tuple[str | bytes, Any]]
|
|
||||||
| None = None,
|
|
||||||
body: bytes | str | None = None,
|
body: bytes | str | None = None,
|
||||||
cookies: CookiesT | None = None,
|
cookies: CookiesT | None = None,
|
||||||
meta: dict[str, Any] | None = None,
|
meta: dict[str, Any] | None = None,
|
||||||
|
|
|
||||||
|
|
@ -1,23 +1,19 @@
|
||||||
# pragma: no file cover
|
|
||||||
# pylint: disable=no-method-argument,no-self-argument
|
# pylint: disable=no-method-argument,no-self-argument
|
||||||
import warnings
|
|
||||||
|
|
||||||
from zope.interface import Interface
|
from zope.interface import Interface
|
||||||
|
|
||||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
|
||||||
|
|
||||||
warnings.warn(
|
|
||||||
"The scrapy.interfaces module is deprecated.",
|
|
||||||
ScrapyDeprecationWarning,
|
|
||||||
stacklevel=2,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class ISpiderLoader(Interface):
|
class ISpiderLoader(Interface):
|
||||||
def from_settings(settings): ...
|
def from_settings(settings):
|
||||||
|
"""Return an instance of the class for the given settings"""
|
||||||
|
|
||||||
def load(spider_name): ...
|
def load(spider_name):
|
||||||
|
"""Return the Spider class for the given spider name. If the spider
|
||||||
|
name is not found, it must raise a KeyError."""
|
||||||
|
|
||||||
def list(): ...
|
def list():
|
||||||
|
"""Return a list with the names of all spiders available in the
|
||||||
|
project"""
|
||||||
|
|
||||||
def find_by_request(request): ...
|
def find_by_request(request):
|
||||||
|
"""Return the list of spiders names that can handle the given request"""
|
||||||
|
|
|
||||||
|
|
@ -2,8 +2,6 @@
|
||||||
Mail sending helpers
|
Mail sending helpers
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# pragma: no file cover
|
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
|
||||||
|
|
@ -27,17 +27,9 @@ from twisted.internet.defer import Deferred, maybeDeferred
|
||||||
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
|
from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
|
||||||
from scrapy.http import Request, Response
|
from scrapy.http import Request, Response
|
||||||
from scrapy.http.request import NO_CALLBACK
|
from scrapy.http.request import NO_CALLBACK
|
||||||
from scrapy.pipelines.media import (
|
from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline
|
||||||
FileException as FileException, # noqa: PLC0414 # re-exported for backward compatibility
|
|
||||||
)
|
|
||||||
from scrapy.pipelines.media import (
|
|
||||||
FileInfo,
|
|
||||||
FileInfoOrError,
|
|
||||||
MediaPipeline,
|
|
||||||
_MediaRequestFiltered,
|
|
||||||
)
|
|
||||||
from scrapy.utils.asyncio import run_in_thread
|
from scrapy.utils.asyncio import run_in_thread
|
||||||
from scrapy.utils.boto import _get_max_pool_connections, is_botocore_available
|
from scrapy.utils.boto import is_botocore_available
|
||||||
from scrapy.utils.datatypes import CaseInsensitiveDict
|
from scrapy.utils.datatypes import CaseInsensitiveDict
|
||||||
from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
|
from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
|
||||||
from scrapy.utils.ftp import ftp_store_file
|
from scrapy.utils.ftp import ftp_store_file
|
||||||
|
|
@ -83,6 +75,10 @@ def _md5sum(file: IO[bytes]) -> str:
|
||||||
return m.hexdigest()
|
return m.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
class FileException(Exception):
|
||||||
|
"""General media error exception"""
|
||||||
|
|
||||||
|
|
||||||
class StatInfo(TypedDict, total=False):
|
class StatInfo(TypedDict, total=False):
|
||||||
checksum: str
|
checksum: str
|
||||||
last_modified: float
|
last_modified: float
|
||||||
|
|
@ -164,9 +160,6 @@ class S3FilesStore:
|
||||||
AWS_REGION_NAME = None
|
AWS_REGION_NAME = None
|
||||||
AWS_USE_SSL = None
|
AWS_USE_SSL = None
|
||||||
AWS_VERIFY = None
|
AWS_VERIFY = None
|
||||||
# Overridden from settings.AWS_MAX_POOL_CONNECTIONS in
|
|
||||||
# FilesPipeline.from_crawler(); None means the botocore default
|
|
||||||
AWS_MAX_POOL_CONNECTIONS: int | None = None
|
|
||||||
|
|
||||||
POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler()
|
POLICY = "private" # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler()
|
||||||
HEADERS: ClassVar[dict[str, str]] = {
|
HEADERS: ClassVar[dict[str, str]] = {
|
||||||
|
|
@ -177,13 +170,7 @@ class S3FilesStore:
|
||||||
if not is_botocore_available():
|
if not is_botocore_available():
|
||||||
raise NotConfigured("missing botocore library")
|
raise NotConfigured("missing botocore library")
|
||||||
import botocore.session # noqa: PLC0415
|
import botocore.session # noqa: PLC0415
|
||||||
from botocore.config import Config # noqa: PLC0415
|
|
||||||
|
|
||||||
config = (
|
|
||||||
Config(max_pool_connections=self.AWS_MAX_POOL_CONNECTIONS)
|
|
||||||
if self.AWS_MAX_POOL_CONNECTIONS is not None
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
session = botocore.session.get_session()
|
session = botocore.session.get_session()
|
||||||
self.s3_client = session.create_client(
|
self.s3_client = session.create_client(
|
||||||
"s3",
|
"s3",
|
||||||
|
|
@ -194,7 +181,6 @@ class S3FilesStore:
|
||||||
region_name=self.AWS_REGION_NAME,
|
region_name=self.AWS_REGION_NAME,
|
||||||
use_ssl=self.AWS_USE_SSL,
|
use_ssl=self.AWS_USE_SSL,
|
||||||
verify=self.AWS_VERIFY,
|
verify=self.AWS_VERIFY,
|
||||||
config=config,
|
|
||||||
)
|
)
|
||||||
if not uri.startswith("s3://"):
|
if not uri.startswith("s3://"):
|
||||||
raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
|
raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
|
||||||
|
|
@ -314,13 +300,13 @@ class GCSFilesStore:
|
||||||
)
|
)
|
||||||
if "storage.objects.get" not in permissions:
|
if "storage.objects.get" not in permissions:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"No 'storage.objects.get' permission for GCS bucket %(bucket)s. "
|
"No 'storage.objects.get' permission for GSC bucket %(bucket)s. "
|
||||||
"Checking if files are up to date will be impossible. Files will be downloaded every time.",
|
"Checking if files are up to date will be impossible. Files will be downloaded every time.",
|
||||||
{"bucket": bucket},
|
{"bucket": bucket},
|
||||||
)
|
)
|
||||||
if "storage.objects.create" not in permissions:
|
if "storage.objects.create" not in permissions:
|
||||||
logger.error(
|
logger.error(
|
||||||
"No 'storage.objects.create' permission for GCS bucket %(bucket)s. Saving files will be impossible!",
|
"No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!",
|
||||||
{"bucket": bucket},
|
{"bucket": bucket},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
@ -532,7 +518,6 @@ class FilesPipeline(MediaPipeline):
|
||||||
s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"]
|
s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"]
|
||||||
s3store.AWS_USE_SSL = settings["AWS_USE_SSL"]
|
s3store.AWS_USE_SSL = settings["AWS_USE_SSL"]
|
||||||
s3store.AWS_VERIFY = settings["AWS_VERIFY"]
|
s3store.AWS_VERIFY = settings["AWS_VERIFY"]
|
||||||
s3store.AWS_MAX_POOL_CONNECTIONS = _get_max_pool_connections(settings)
|
|
||||||
s3store.POLICY = settings["FILES_STORE_S3_ACL"]
|
s3store.POLICY = settings["FILES_STORE_S3_ACL"]
|
||||||
|
|
||||||
gcs_store: type[GCSFilesStore] = cast(
|
gcs_store: type[GCSFilesStore] = cast(
|
||||||
|
|
@ -612,20 +597,20 @@ class FilesPipeline(MediaPipeline):
|
||||||
def media_failed(
|
def media_failed(
|
||||||
self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
|
self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
|
||||||
) -> NoReturn:
|
) -> NoReturn:
|
||||||
referer = referer_str(request)
|
if not isinstance(failure.value, IgnoreRequest):
|
||||||
if isinstance(failure.value, IgnoreRequest):
|
referer = referer_str(request)
|
||||||
logger.debug(
|
logger.warning(
|
||||||
f"File (filtered): Not downloading {self.MEDIA_NAME} from "
|
"File (unknown-error): Error downloading %(medianame)s from "
|
||||||
f"{request} referred in <{referer}>: {failure.value}",
|
"%(request)s referred in <%(referer)s>: %(exception)s",
|
||||||
|
{
|
||||||
|
"medianame": self.MEDIA_NAME,
|
||||||
|
"request": request,
|
||||||
|
"referer": referer,
|
||||||
|
"exception": failure.value,
|
||||||
|
},
|
||||||
extra={"spider": info.spider},
|
extra={"spider": info.spider},
|
||||||
)
|
)
|
||||||
raise _MediaRequestFiltered(str(failure.value)) from failure.value
|
|
||||||
|
|
||||||
logger.warning(
|
|
||||||
f"File (unknown-error): Error downloading {self.MEDIA_NAME} from "
|
|
||||||
f"{request} referred in <{referer}>: {failure.value}",
|
|
||||||
extra={"spider": info.spider},
|
|
||||||
)
|
|
||||||
raise FileException
|
raise FileException
|
||||||
|
|
||||||
async def media_downloaded(
|
async def media_downloaded(
|
||||||
|
|
|
||||||
|
|
@ -55,25 +55,6 @@ FileInfoOrError: TypeAlias = (
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
class FileException(Exception):
|
|
||||||
"""General media error exception"""
|
|
||||||
|
|
||||||
|
|
||||||
class _MediaRequestFiltered(FileException):
|
|
||||||
"""Raised internally by media pipelines when a media request is filtered
|
|
||||||
out (e.g. as an offsite request) instead of being downloaded.
|
|
||||||
|
|
||||||
It is a subclass of :exc:`FileException` for backward compatibility, but
|
|
||||||
unlike an actual download error it is logged at the ``DEBUG`` level and
|
|
||||||
without a traceback, since filtering a request is expected behavior rather
|
|
||||||
than an error.
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def _media_request_filtered(failure: Failure) -> bool:
|
|
||||||
return isinstance(failure.value, _MediaRequestFiltered)
|
|
||||||
|
|
||||||
|
|
||||||
class MediaPipeline(ABC):
|
class MediaPipeline(ABC):
|
||||||
LOG_FAILED_RESULTS: bool = True
|
LOG_FAILED_RESULTS: bool = True
|
||||||
|
|
||||||
|
|
@ -212,8 +193,7 @@ class MediaPipeline(ABC):
|
||||||
result = await self._check_media_to_download(request, info, item=item)
|
result = await self._check_media_to_download(request, info, item=item)
|
||||||
except Exception:
|
except Exception:
|
||||||
result = Failure()
|
result = Failure()
|
||||||
if not _media_request_filtered(result):
|
logger.exception(result)
|
||||||
logger.exception(result)
|
|
||||||
self._cache_result_and_execute_waiters(result, fp, info)
|
self._cache_result_and_execute_waiters(result, fp, info)
|
||||||
return await maybe_deferred_to_future(wad) # it must return wad at last
|
return await maybe_deferred_to_future(wad) # it must return wad at last
|
||||||
|
|
||||||
|
|
@ -324,8 +304,6 @@ class MediaPipeline(ABC):
|
||||||
for ok, value in results:
|
for ok, value in results:
|
||||||
if not ok:
|
if not ok:
|
||||||
assert isinstance(value, Failure)
|
assert isinstance(value, Failure)
|
||||||
if _media_request_filtered(value):
|
|
||||||
continue
|
|
||||||
logger.error(
|
logger.error(
|
||||||
"%(class)s found errors processing %(item)s",
|
"%(class)s found errors processing %(item)s",
|
||||||
{"class": self.__class__.__name__, "item": item},
|
{"class": self.__class__.__name__, "item": item},
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,6 @@ from __future__ import annotations
|
||||||
|
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
import attr
|
|
||||||
from twisted.internet import defer
|
from twisted.internet import defer
|
||||||
from twisted.internet.base import ReactorBase, ThreadedResolver
|
from twisted.internet.base import ReactorBase, ThreadedResolver
|
||||||
from twisted.internet.interfaces import (
|
from twisted.internet.interfaces import (
|
||||||
|
|
@ -71,12 +70,6 @@ class CachingThreadedResolver(ThreadedResolver):
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def _address_with_port(address: IAddress, port: int) -> IAddress:
|
|
||||||
if getattr(address, "port", port) == port:
|
|
||||||
return address
|
|
||||||
return attr.evolve(address, port=port)
|
|
||||||
|
|
||||||
|
|
||||||
@implementer(IHostResolution)
|
@implementer(IHostResolution)
|
||||||
class HostResolution:
|
class HostResolution:
|
||||||
def __init__(self, name: str):
|
def __init__(self, name: str):
|
||||||
|
|
@ -104,11 +97,7 @@ class _CachingResolutionReceiver:
|
||||||
def resolutionComplete(self) -> None:
|
def resolutionComplete(self) -> None:
|
||||||
self.resolutionReceiver.resolutionComplete()
|
self.resolutionReceiver.resolutionComplete()
|
||||||
if self.addresses:
|
if self.addresses:
|
||||||
# Name resolution does not depend on the port, so cache entries are
|
dnscache[self.hostName] = self.addresses
|
||||||
# kept port-agnostic and the requested port is set on cache hits.
|
|
||||||
dnscache[self.hostName] = [
|
|
||||||
_address_with_port(address, 0) for address in self.addresses
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@implementer(IHostnameResolver)
|
@implementer(IHostnameResolver)
|
||||||
|
|
@ -153,7 +142,7 @@ class CachingHostnameResolver:
|
||||||
transportSemantics,
|
transportSemantics,
|
||||||
)
|
)
|
||||||
resolutionReceiver.resolutionBegan(HostResolution(hostName))
|
resolutionReceiver.resolutionBegan(HostResolution(hostName))
|
||||||
for address in addresses:
|
for addr in addresses:
|
||||||
resolutionReceiver.addressResolved(_address_with_port(address, portNumber))
|
resolutionReceiver.addressResolved(addr)
|
||||||
resolutionReceiver.resolutionComplete()
|
resolutionReceiver.resolutionComplete()
|
||||||
return resolutionReceiver
|
return resolutionReceiver
|
||||||
|
|
|
||||||
|
|
@ -67,15 +67,6 @@ class RobotParser(metaclass=ABCMeta):
|
||||||
:type user_agent: str or bytes
|
:type user_agent: str or bytes
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def crawl_delay(self, user_agent: str | bytes) -> float | None:
|
|
||||||
"""Return the ``Crawl-delay`` directive for ``user_agent`` as a number
|
|
||||||
of seconds, or ``None`` if it is not set or the backend does not support
|
|
||||||
it.
|
|
||||||
|
|
||||||
.. versionadded:: VERSION
|
|
||||||
"""
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
class PythonRobotParser(RobotParser):
|
class PythonRobotParser(RobotParser):
|
||||||
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
|
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
|
||||||
|
|
@ -94,10 +85,6 @@ class PythonRobotParser(RobotParser):
|
||||||
url = to_unicode(url)
|
url = to_unicode(url)
|
||||||
return self.rp.can_fetch(user_agent, url)
|
return self.rp.can_fetch(user_agent, url)
|
||||||
|
|
||||||
def crawl_delay(self, user_agent: str | bytes) -> float | None:
|
|
||||||
delay = self.rp.crawl_delay(to_unicode(user_agent))
|
|
||||||
return None if delay is None else float(delay)
|
|
||||||
|
|
||||||
|
|
||||||
class RerpRobotParser(RobotParser):
|
class RerpRobotParser(RobotParser):
|
||||||
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
|
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
|
||||||
|
|
@ -118,10 +105,6 @@ class RerpRobotParser(RobotParser):
|
||||||
url = to_unicode(url)
|
url = to_unicode(url)
|
||||||
return cast("bool", self.rp.is_allowed(user_agent, url))
|
return cast("bool", self.rp.is_allowed(user_agent, url))
|
||||||
|
|
||||||
def crawl_delay(self, user_agent: str | bytes) -> float | None:
|
|
||||||
delay = self.rp.get_crawl_delay(to_unicode(user_agent))
|
|
||||||
return None if delay is None else float(delay)
|
|
||||||
|
|
||||||
|
|
||||||
class ProtegoRobotParser(RobotParser):
|
class ProtegoRobotParser(RobotParser):
|
||||||
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
|
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
|
||||||
|
|
@ -138,7 +121,3 @@ class ProtegoRobotParser(RobotParser):
|
||||||
user_agent = to_unicode(user_agent)
|
user_agent = to_unicode(user_agent)
|
||||||
url = to_unicode(url)
|
url = to_unicode(url)
|
||||||
return self.rp.can_fetch(url, user_agent)
|
return self.rp.can_fetch(url, user_agent)
|
||||||
|
|
||||||
def crawl_delay(self, user_agent: str | bytes) -> float | None:
|
|
||||||
delay = self.rp.crawl_delay(to_unicode(user_agent))
|
|
||||||
return None if delay is None else float(delay)
|
|
||||||
|
|
|
||||||
|
|
@ -25,9 +25,7 @@ if TYPE_CHECKING:
|
||||||
# typing.Self requires Python 3.11
|
# typing.Self requires Python 3.11
|
||||||
from typing_extensions import Self
|
from typing_extensions import Self
|
||||||
|
|
||||||
_SettingsInput: TypeAlias = (
|
_SettingsInput: TypeAlias = SupportsItems[str, Any] | str | None
|
||||||
SupportsItems[str, Any] | Iterable[tuple[str, Any]] | str | None
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
SETTINGS_PRIORITIES: dict[str, int] = {
|
SETTINGS_PRIORITIES: dict[str, int] = {
|
||||||
|
|
@ -290,10 +288,10 @@ class BaseSettings(MutableMapping[str, Any]):
|
||||||
- ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
|
- ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
|
||||||
|
|
||||||
:param name: the setting name
|
:param name: the setting name
|
||||||
:type name: str
|
:type name: string
|
||||||
|
|
||||||
:param default: the value to return if no setting is found
|
:param default: the value to return if no setting is found
|
||||||
:type default: object
|
:type default: any
|
||||||
"""
|
"""
|
||||||
value = self.get(name, default)
|
value = self.get(name, default)
|
||||||
if value is None:
|
if value is None:
|
||||||
|
|
@ -562,7 +560,7 @@ class BaseSettings(MutableMapping[str, Any]):
|
||||||
if key.isupper():
|
if key.isupper():
|
||||||
self.set(key, getattr(module, key), priority)
|
self.set(key, getattr(module, key), priority)
|
||||||
|
|
||||||
# BaseSettings.update() doesn't support kwargs input like MutableMapping.update().
|
# BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports
|
||||||
def update(self, values: _SettingsInput, priority: int | str = "project") -> None: # type: ignore[override]
|
def update(self, values: _SettingsInput, priority: int | str = "project") -> None: # type: ignore[override]
|
||||||
"""
|
"""
|
||||||
Store key/value pairs with a given priority.
|
Store key/value pairs with a given priority.
|
||||||
|
|
@ -579,7 +577,7 @@ class BaseSettings(MutableMapping[str, Any]):
|
||||||
command.
|
command.
|
||||||
|
|
||||||
:param values: the settings names and values
|
:param values: the settings names and values
|
||||||
:type values: dict, iterable, string or :class:`~scrapy.settings.BaseSettings`
|
:type values: dict or string or :class:`~scrapy.settings.BaseSettings`
|
||||||
|
|
||||||
:param priority: the priority of the settings. Should be a key of
|
:param priority: the priority of the settings. Should be a key of
|
||||||
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
|
:attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
|
||||||
|
|
@ -593,12 +591,7 @@ class BaseSettings(MutableMapping[str, Any]):
|
||||||
for name, value in values.items():
|
for name, value in values.items():
|
||||||
self.set(name, value, cast("int", values.getpriority(name)))
|
self.set(name, value, cast("int", values.getpriority(name)))
|
||||||
else:
|
else:
|
||||||
items: Iterable[tuple[str, Any]]
|
for name, value in values.items():
|
||||||
if hasattr(values, "items"):
|
|
||||||
items = cast("SupportsItems[str, Any]", values).items()
|
|
||||||
else:
|
|
||||||
items = values
|
|
||||||
for name, value in items:
|
|
||||||
self.set(name, value, priority)
|
self.set(name, value, priority)
|
||||||
|
|
||||||
def delete(self, name: str, priority: int | str = "project") -> None:
|
def delete(self, name: str, priority: int | str = "project") -> None:
|
||||||
|
|
|
||||||
|
|
@ -28,7 +28,6 @@ __all__ = [
|
||||||
"AUTOTHROTTLE_TARGET_CONCURRENCY",
|
"AUTOTHROTTLE_TARGET_CONCURRENCY",
|
||||||
"AWS_ACCESS_KEY_ID",
|
"AWS_ACCESS_KEY_ID",
|
||||||
"AWS_ENDPOINT_URL",
|
"AWS_ENDPOINT_URL",
|
||||||
"AWS_MAX_POOL_CONNECTIONS",
|
|
||||||
"AWS_REGION_NAME",
|
"AWS_REGION_NAME",
|
||||||
"AWS_SECRET_ACCESS_KEY",
|
"AWS_SECRET_ACCESS_KEY",
|
||||||
"AWS_SESSION_TOKEN",
|
"AWS_SESSION_TOKEN",
|
||||||
|
|
@ -155,6 +154,7 @@ __all__ = [
|
||||||
"MAIL_TLS",
|
"MAIL_TLS",
|
||||||
"MAIL_USER",
|
"MAIL_USER",
|
||||||
"MEMDEBUG_ENABLED",
|
"MEMDEBUG_ENABLED",
|
||||||
|
"MEMDEBUG_NOTIFY",
|
||||||
"MEMUSAGE_CHECK_INTERVAL_SECONDS",
|
"MEMUSAGE_CHECK_INTERVAL_SECONDS",
|
||||||
"MEMUSAGE_ENABLED",
|
"MEMUSAGE_ENABLED",
|
||||||
"MEMUSAGE_LIMIT_MB",
|
"MEMUSAGE_LIMIT_MB",
|
||||||
|
|
@ -230,7 +230,6 @@ AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
|
||||||
AWS_ACCESS_KEY_ID = None
|
AWS_ACCESS_KEY_ID = None
|
||||||
AWS_SECRET_ACCESS_KEY = None
|
AWS_SECRET_ACCESS_KEY = None
|
||||||
AWS_ENDPOINT_URL = None
|
AWS_ENDPOINT_URL = None
|
||||||
AWS_MAX_POOL_CONNECTIONS = None
|
|
||||||
AWS_REGION_NAME = None
|
AWS_REGION_NAME = None
|
||||||
AWS_SESSION_TOKEN = None
|
AWS_SESSION_TOKEN = None
|
||||||
AWS_USE_SSL = None
|
AWS_USE_SSL = None
|
||||||
|
|
@ -241,7 +240,7 @@ BOT_NAME = "scrapybot"
|
||||||
CLOSESPIDER_ERRORCOUNT = 0
|
CLOSESPIDER_ERRORCOUNT = 0
|
||||||
CLOSESPIDER_ITEMCOUNT = 0
|
CLOSESPIDER_ITEMCOUNT = 0
|
||||||
CLOSESPIDER_PAGECOUNT = 0
|
CLOSESPIDER_PAGECOUNT = 0
|
||||||
CLOSESPIDER_TIMEOUT = 0.0
|
CLOSESPIDER_TIMEOUT = 0
|
||||||
CLOSESPIDER_PAGECOUNT_NO_ITEM = 0
|
CLOSESPIDER_PAGECOUNT_NO_ITEM = 0
|
||||||
CLOSESPIDER_TIMEOUT_NO_ITEM = 0
|
CLOSESPIDER_TIMEOUT_NO_ITEM = 0
|
||||||
|
|
||||||
|
|
@ -471,6 +470,7 @@ MAIL_SSL = False
|
||||||
MAIL_TLS = False
|
MAIL_TLS = False
|
||||||
|
|
||||||
MEMDEBUG_ENABLED = False # enable memory debugging
|
MEMDEBUG_ENABLED = False # enable memory debugging
|
||||||
|
MEMDEBUG_NOTIFY = [] # send memory debugging report by mail at engine shutdown
|
||||||
|
|
||||||
MEMUSAGE_ENABLED = True
|
MEMUSAGE_ENABLED = True
|
||||||
MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
|
MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
|
||||||
|
|
@ -556,7 +556,6 @@ SPIDER_MIDDLEWARES_BASE = {
|
||||||
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
|
"scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
|
||||||
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
|
"scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
|
||||||
"scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
|
"scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
|
||||||
"scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware": 1000,
|
|
||||||
# Spider side
|
# Spider side
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,6 @@ response_received = object()
|
||||||
response_downloaded = object()
|
response_downloaded = object()
|
||||||
headers_received = object()
|
headers_received = object()
|
||||||
bytes_received = object()
|
bytes_received = object()
|
||||||
robots_parsed = object()
|
|
||||||
item_scraped = object()
|
item_scraped = object()
|
||||||
item_dropped = object()
|
item_dropped = object()
|
||||||
item_error = object()
|
item_error = object()
|
||||||
|
|
|
||||||
|
|
@ -5,8 +5,10 @@ import warnings
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from typing import TYPE_CHECKING, Protocol, cast
|
from typing import TYPE_CHECKING, Protocol, cast
|
||||||
|
|
||||||
# working around https://github.com/sphinx-doc/sphinx/issues/10400
|
from zope.interface import implementer
|
||||||
from scrapy import Request, Spider # noqa: TC001
|
from zope.interface.verify import verifyClass
|
||||||
|
|
||||||
|
from scrapy.interfaces import ISpiderLoader
|
||||||
from scrapy.utils.misc import load_object, walk_modules_iter
|
from scrapy.utils.misc import load_object, walk_modules_iter
|
||||||
from scrapy.utils.spider import iter_spider_classes
|
from scrapy.utils.spider import iter_spider_classes
|
||||||
|
|
||||||
|
|
@ -16,6 +18,7 @@ if TYPE_CHECKING:
|
||||||
# typing.Self requires Python 3.11
|
# typing.Self requires Python 3.11
|
||||||
from typing_extensions import Self
|
from typing_extensions import Self
|
||||||
|
|
||||||
|
from scrapy import Request, Spider
|
||||||
from scrapy.settings import BaseSettings
|
from scrapy.settings import BaseSettings
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -23,31 +26,28 @@ def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:
|
||||||
"""Get SpiderLoader instance from settings"""
|
"""Get SpiderLoader instance from settings"""
|
||||||
cls_path = settings.get("SPIDER_LOADER_CLASS")
|
cls_path = settings.get("SPIDER_LOADER_CLASS")
|
||||||
loader_cls = load_object(cls_path)
|
loader_cls = load_object(cls_path)
|
||||||
|
verifyClass(ISpiderLoader, loader_cls)
|
||||||
return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()))
|
return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()))
|
||||||
|
|
||||||
|
|
||||||
class SpiderLoaderProtocol(Protocol):
|
class SpiderLoaderProtocol(Protocol):
|
||||||
"""Protocol for spider loader implementations.
|
|
||||||
|
|
||||||
See :setting:`SPIDER_LOADER_CLASS`.
|
|
||||||
"""
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_settings(cls, settings: BaseSettings) -> Self:
|
def from_settings(cls, settings: BaseSettings) -> Self:
|
||||||
"""Return an instance of the class for the given settings."""
|
"""Return an instance of the class for the given settings"""
|
||||||
|
|
||||||
def load(self, spider_name: str) -> type[Spider]:
|
def load(self, spider_name: str) -> type[Spider]:
|
||||||
"""Return the spider class for the given spider name. If the spider
|
"""Return the Spider class for the given spider name. If the spider
|
||||||
name is not found, it must raise a :exc:`KeyError`."""
|
name is not found, it must raise a KeyError."""
|
||||||
|
|
||||||
def list(self) -> list[str]:
|
def list(self) -> list[str]:
|
||||||
"""Return a list with the names of all spiders available in the
|
"""Return a list with the names of all spiders available in the
|
||||||
project."""
|
project"""
|
||||||
|
|
||||||
def find_by_request(self, request: Request) -> __builtins__.list[str]:
|
def find_by_request(self, request: Request) -> __builtins__.list[str]:
|
||||||
"""Return the list of spiders names that can handle the given request."""
|
"""Return the list of spiders names that can handle the given request"""
|
||||||
|
|
||||||
|
|
||||||
|
@implementer(ISpiderLoader)
|
||||||
class SpiderLoader:
|
class SpiderLoader:
|
||||||
"""
|
"""
|
||||||
SpiderLoader is a class which locates and loads spiders
|
SpiderLoader is a class which locates and loads spiders
|
||||||
|
|
@ -106,18 +106,12 @@ class SpiderLoader:
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def from_settings(cls, settings: BaseSettings) -> Self:
|
def from_settings(cls, settings: BaseSettings) -> Self:
|
||||||
"""Create an instance of the class.
|
|
||||||
|
|
||||||
It's called with the current project settings, and it loads the spiders
|
|
||||||
found recursively in the modules of the :setting:`SPIDER_MODULES`
|
|
||||||
setting.
|
|
||||||
"""
|
|
||||||
return cls(settings)
|
return cls(settings)
|
||||||
|
|
||||||
def load(self, spider_name: str) -> type[Spider]:
|
def load(self, spider_name: str) -> type[Spider]:
|
||||||
"""Return the spider class for the given spider name.
|
"""
|
||||||
|
Return the Spider class for the given spider name. If the spider
|
||||||
If the spider name is not found, raise a :exc:`KeyError`.
|
name is not found, raise a KeyError.
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
return self._spiders[spider_name]
|
return self._spiders[spider_name]
|
||||||
|
|
@ -127,19 +121,19 @@ class SpiderLoader:
|
||||||
def find_by_request(self, request: Request) -> list[str]:
|
def find_by_request(self, request: Request) -> list[str]:
|
||||||
"""
|
"""
|
||||||
Return the list of spider names that can handle the given request.
|
Return the list of spider names that can handle the given request.
|
||||||
|
|
||||||
It will try to match the request's url against the domains of
|
|
||||||
the spiders.
|
|
||||||
"""
|
"""
|
||||||
return [
|
return [
|
||||||
name for name, cls in self._spiders.items() if cls.handles_request(request)
|
name for name, cls in self._spiders.items() if cls.handles_request(request)
|
||||||
]
|
]
|
||||||
|
|
||||||
def list(self) -> list[str]:
|
def list(self) -> list[str]:
|
||||||
"""Return a list with the names of all spiders available in the project."""
|
"""
|
||||||
|
Return a list with the names of all spiders available in the project.
|
||||||
|
"""
|
||||||
return list(self._spiders.keys())
|
return list(self._spiders.keys())
|
||||||
|
|
||||||
|
|
||||||
|
@implementer(ISpiderLoader)
|
||||||
class DummySpiderLoader:
|
class DummySpiderLoader:
|
||||||
"""A dummy spider loader that does not load any spiders."""
|
"""A dummy spider loader that does not load any spiders."""
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -48,10 +48,7 @@ class BaseSpiderMiddleware:
|
||||||
|
|
||||||
@_warn_spider_arg
|
@_warn_spider_arg
|
||||||
def process_spider_output(
|
def process_spider_output(
|
||||||
self,
|
self, response: Response, result: Iterable[Any], spider: Spider | None = None
|
||||||
response: Response | None,
|
|
||||||
result: Iterable[Any],
|
|
||||||
spider: Spider | None = None,
|
|
||||||
) -> Iterable[Any]:
|
) -> Iterable[Any]:
|
||||||
for o in result:
|
for o in result:
|
||||||
if (o := self._get_processed(o, response)) is not None:
|
if (o := self._get_processed(o, response)) is not None:
|
||||||
|
|
@ -60,7 +57,7 @@ class BaseSpiderMiddleware:
|
||||||
@_warn_spider_arg
|
@_warn_spider_arg
|
||||||
async def process_spider_output_async(
|
async def process_spider_output_async(
|
||||||
self,
|
self,
|
||||||
response: Response | None,
|
response: Response,
|
||||||
result: AsyncIterator[Any],
|
result: AsyncIterator[Any],
|
||||||
spider: Spider | None = None,
|
spider: Spider | None = None,
|
||||||
) -> AsyncIterator[Any]:
|
) -> AsyncIterator[Any]:
|
||||||
|
|
|
||||||
|
|
@ -55,24 +55,19 @@ class DepthMiddleware(BaseSpiderMiddleware):
|
||||||
|
|
||||||
@_warn_spider_arg
|
@_warn_spider_arg
|
||||||
def process_spider_output(
|
def process_spider_output(
|
||||||
self,
|
self, response: Response, result: Iterable[Any], spider: Spider | None = None
|
||||||
response: Response | None,
|
|
||||||
result: Iterable[Any],
|
|
||||||
spider: Spider | None = None,
|
|
||||||
) -> Iterable[Any]:
|
) -> Iterable[Any]:
|
||||||
if response is not None:
|
self._init_depth(response)
|
||||||
self._init_depth(response)
|
|
||||||
yield from super().process_spider_output(response, result)
|
yield from super().process_spider_output(response, result)
|
||||||
|
|
||||||
@_warn_spider_arg
|
@_warn_spider_arg
|
||||||
async def process_spider_output_async(
|
async def process_spider_output_async(
|
||||||
self,
|
self,
|
||||||
response: Response | None,
|
response: Response,
|
||||||
result: AsyncIterator[Any],
|
result: AsyncIterator[Any],
|
||||||
spider: Spider | None = None,
|
spider: Spider | None = None,
|
||||||
) -> AsyncIterator[Any]:
|
) -> AsyncIterator[Any]:
|
||||||
if response is not None:
|
self._init_depth(response)
|
||||||
self._init_depth(response)
|
|
||||||
async for o in super().process_spider_output_async(response, result):
|
async for o in super().process_spider_output_async(response, result):
|
||||||
yield o
|
yield o
|
||||||
|
|
||||||
|
|
|
||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue