mirror of https://github.com/scrapy/scrapy.git
Merge branch 'master' into mcp-backend
This commit is contained in:
Andrey Rakhmatullin
commit
5fa850c70e
|
|
@ -0,0 +1,12 @@
|
|||
version: 2
|
||||
updates:
|
||||
- package-ecosystem: github-actions
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: monthly
|
||||
groups:
|
||||
github-actions:
|
||||
patterns:
|
||||
- "*"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
name: Auto-close LLM PRs
|
||||
on:
|
||||
# The workflow only reads the pull request body through the API, it never
|
||||
# checks out or runs pull request code, so pull_request_target is safe here.
|
||||
on: # zizmor: ignore[dangerous-triggers]
|
||||
pull_request_target:
|
||||
types: [opened]
|
||||
permissions:
|
||||
|
|
@ -11,7 +13,7 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Check PR body and close if LLM-written
|
||||
uses: actions/github-script@v6
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
name: Checks
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
|
|
@ -13,6 +17,10 @@ concurrency:
|
|||
jobs:
|
||||
checks:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# Make uv use the interpreter that actions/setup-python installed instead
|
||||
# of downloading one of its own.
|
||||
UV_PYTHON_PREFERENCE: only-system
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
|
|
@ -38,21 +46,31 @@ jobs:
|
|||
TOXENV: twinecheck
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
with:
|
||||
cache-dependency-glob: |
|
||||
docs/requirements.txt
|
||||
pyproject.toml
|
||||
tox.ini
|
||||
|
||||
- name: Run check
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
pip install -U tox
|
||||
tox
|
||||
run: uvx --with tox-uv tox
|
||||
|
||||
pre-commit:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: pre-commit/action@v3.0.1
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
|
||||
|
|
|
|||
|
|
@ -22,24 +22,38 @@ permissions: {}
|
|||
jobs:
|
||||
benchmark:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# Make uv use the interpreter that actions/setup-python installed
|
||||
# instead of downloading one of its own.
|
||||
UV_PYTHON_PREFERENCE: only-system
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write # OIDC authentication with CodSpeed
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python 3.14
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: '3.14'
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
with:
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
tox.ini
|
||||
|
||||
- name: Install dependencies
|
||||
# tox must stay on PATH for the CodSpeed action to invoke it.
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install --upgrade tox
|
||||
uv tool install --with tox-uv tox
|
||||
tox -n -e benchmark
|
||||
- name: Run benchmarks
|
||||
uses: CodSpeedHQ/action@v4
|
||||
uses: CodSpeedHQ/action@f22792bfac16f3e14eb9fbea76f4a48e9cc22b93 # v4.19.1
|
||||
with:
|
||||
mode: simulation
|
||||
run: tox -e benchmark
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
name: Publish
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
|
|
@ -9,8 +13,28 @@ concurrency:
|
|||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: Build distribution
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- run: |
|
||||
python -m pip install --upgrade build
|
||||
python -m build
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: python-package-distributions
|
||||
path: dist/
|
||||
|
||||
publish:
|
||||
name: Upload release to PyPI
|
||||
needs:
|
||||
- build
|
||||
runs-on: ubuntu-latest
|
||||
environment:
|
||||
name: pypi
|
||||
|
|
@ -18,12 +42,9 @@ jobs:
|
|||
permissions:
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/setup-python@v6
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
python-version: "3.14"
|
||||
- run: |
|
||||
python -m pip install --upgrade build
|
||||
python -m build
|
||||
name: python-package-distributions
|
||||
path: dist/
|
||||
- name: Publish to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
name: macOS
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
|
|
@ -12,44 +16,62 @@ concurrency:
|
|||
|
||||
jobs:
|
||||
tests:
|
||||
name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})
|
||||
runs-on: macos-latest
|
||||
env:
|
||||
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
|
||||
# Make uv use the interpreter that actions/setup-python installed instead
|
||||
# of downloading one of its own.
|
||||
UV_PYTHON_PREFERENCE: only-system
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
||||
env:
|
||||
- TOXENV: py
|
||||
include:
|
||||
- python-version: '3.14'
|
||||
env:
|
||||
TOXENV: py
|
||||
coverage: true
|
||||
- python-version: '3.14'
|
||||
env:
|
||||
TOXENV: no-reactor
|
||||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.14"
|
||||
env:
|
||||
TOXENV: py
|
||||
coverage: true
|
||||
- python-version: "3.14"
|
||||
env:
|
||||
TOXENV: no-reactor
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
with:
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
tox.ini
|
||||
|
||||
- name: Install mitmproxy
|
||||
env:
|
||||
# mitmproxy needs a newer Python than the oldest matrix entries, so let
|
||||
# uv download one where no system interpreter is new enough.
|
||||
UV_PYTHON_PREFERENCE: system
|
||||
run: uv tool install mitmproxy
|
||||
|
||||
- name: Run tests
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
pip install -U tox
|
||||
tox
|
||||
run: uvx --with tox-uv tox
|
||||
|
||||
- name: Upload coverage report
|
||||
if: ${{ matrix.coverage }}
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
|
||||
- name: Upload test results
|
||||
if: ${{ !cancelled() }}
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
with:
|
||||
report_type: test_results
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
name: Ubuntu
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
|
|
@ -12,9 +16,13 @@ concurrency:
|
|||
|
||||
jobs:
|
||||
tests:
|
||||
name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
|
||||
# Make uv use the interpreter that actions/setup-python installed instead
|
||||
# of downloading one of its own.
|
||||
UV_PYTHON_PREFERENCE: only-system
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
|
|
@ -43,10 +51,6 @@ jobs:
|
|||
env:
|
||||
TOXENV: no-reactor
|
||||
coverage: true
|
||||
# pinned due to https://github.com/pypy/pypy/issues/5388
|
||||
- python-version: pypy3.11-7.3.20
|
||||
env:
|
||||
TOXENV: pypy3
|
||||
|
||||
# min deps
|
||||
- python-version: "3.10.19"
|
||||
|
|
@ -57,10 +61,6 @@ jobs:
|
|||
env:
|
||||
TOXENV: min-default-reactor
|
||||
coverage: true
|
||||
- python-version: "3.10.19"
|
||||
env:
|
||||
TOXENV: min-no-reactor
|
||||
coverage: true
|
||||
# pinned due to https://github.com/pypy/pypy/issues/5388
|
||||
- python-version: pypy3.11-7.3.20
|
||||
env:
|
||||
|
|
@ -92,10 +92,12 @@ jobs:
|
|||
coverage: true
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
|
|
@ -105,21 +107,32 @@ jobs:
|
|||
sudo apt-get update
|
||||
sudo apt-get install libxml2-dev libxslt-dev
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
with:
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
tox.ini
|
||||
|
||||
- name: Install mitmproxy
|
||||
run: pipx install mitmproxy
|
||||
env:
|
||||
# mitmproxy needs a newer Python than the oldest matrix entries, so let
|
||||
# uv download one where no system interpreter is new enough.
|
||||
UV_PYTHON_PREFERENCE: system
|
||||
# mitmproxy has no PyPy wheels, so run it on CPython regardless of the
|
||||
# interpreter under test.
|
||||
run: uv tool install --python cpython mitmproxy
|
||||
|
||||
- name: Run tests
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
pip install -U tox
|
||||
tox
|
||||
run: uvx --with tox-uv tox
|
||||
|
||||
- name: Upload coverage report
|
||||
if: ${{ matrix.coverage }}
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
|
||||
- name: Upload test results
|
||||
if: ${{ !cancelled() }}
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
with:
|
||||
report_type: test_results
|
||||
|
|
|
|||
|
|
@ -1,4 +1,8 @@
|
|||
name: Windows
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
|
|
@ -12,9 +16,13 @@ concurrency:
|
|||
|
||||
jobs:
|
||||
tests:
|
||||
name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})
|
||||
runs-on: windows-latest
|
||||
env:
|
||||
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
|
||||
# Make uv use the interpreter that actions/setup-python installed instead
|
||||
# of downloading one of its own.
|
||||
UV_PYTHON_PREFERENCE: only-system
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
|
|
@ -22,22 +30,10 @@ jobs:
|
|||
- python-version: "3.10"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.11"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.12"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.13"
|
||||
env:
|
||||
TOXENV: py
|
||||
- python-version: "3.14"
|
||||
env:
|
||||
TOXENV: py
|
||||
coverage: true
|
||||
- python-version: "3.14"
|
||||
env:
|
||||
TOXENV: default-reactor
|
||||
- python-version: "3.14"
|
||||
env:
|
||||
TOXENV: no-reactor
|
||||
|
|
@ -55,25 +51,39 @@ jobs:
|
|||
TOXENV: extra-deps
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v6
|
||||
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
|
||||
- name: Set up uv
|
||||
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
|
||||
with:
|
||||
cache-dependency-glob: |
|
||||
pyproject.toml
|
||||
tox.ini
|
||||
|
||||
- name: Install mitmproxy
|
||||
env:
|
||||
# mitmproxy needs a newer Python than the oldest matrix entries, so let
|
||||
# uv download one where no system interpreter is new enough.
|
||||
UV_PYTHON_PREFERENCE: system
|
||||
run: uv tool install mitmproxy
|
||||
|
||||
- name: Run tests
|
||||
env: ${{ matrix.env }}
|
||||
run: |
|
||||
pip install -U tox
|
||||
tox
|
||||
run: uvx --with tox-uv tox
|
||||
|
||||
- name: Upload coverage report
|
||||
if: ${{ matrix.coverage }}
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
|
||||
- name: Upload test results
|
||||
if: ${{ !cancelled() }}
|
||||
uses: codecov/codecov-action@v5
|
||||
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
|
||||
with:
|
||||
report_type: test_results
|
||||
|
|
|
|||
|
|
@ -27,6 +27,11 @@ repos:
|
|||
hooks:
|
||||
- id: sphinx-lint
|
||||
- repo: https://github.com/scrapy/sphinx-scrapy
|
||||
rev: 0.8.8
|
||||
rev: 0.8.10
|
||||
hooks:
|
||||
- id: sphinx-scrapy
|
||||
- repo: https://github.com/zizmorcore/zizmor-pre-commit
|
||||
rev: v1.28.0
|
||||
hooks:
|
||||
- id: zizmor
|
||||
args: [--no-progress, --fix]
|
||||
|
|
|
|||
|
|
@ -5,4 +5,4 @@ sphinx
|
|||
sphinx-notfound-page
|
||||
sphinx-rtd-theme
|
||||
sphinx-rtd-dark-mode
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0
|
|||
# via
|
||||
# -r docs/requirements.in
|
||||
# sphinx-rtd-dark-mode
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737
|
||||
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8
|
||||
# via -r docs/requirements.in
|
||||
sphinx-sitemap==2.9.0
|
||||
# via sphinx-scrapy
|
||||
|
|
|
|||
|
|
@ -267,6 +267,7 @@ Here are some examples of APIs and patterns that need a replacement:
|
|||
|
||||
Scrapy provides unified helpers for some of these examples:
|
||||
|
||||
.. autofunction:: scrapy.utils.asyncio.sleep
|
||||
.. autofunction:: scrapy.utils.asyncio.call_later
|
||||
.. autofunction:: scrapy.utils.asyncio.create_looping_call
|
||||
.. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall
|
||||
|
|
|
|||
|
|
@ -9,37 +9,22 @@ A Scrapy component is any class whose objects are built using
|
|||
|
||||
That includes the classes that you may assign to the following settings:
|
||||
|
||||
- :setting:`ADDONS`
|
||||
|
||||
- :setting:`TWISTED_DNS_RESOLVER`
|
||||
|
||||
- :setting:`DOWNLOAD_HANDLERS`
|
||||
|
||||
- :setting:`DOWNLOADER_MIDDLEWARES`
|
||||
|
||||
- :setting:`DUPEFILTER_CLASS`
|
||||
|
||||
- :setting:`EXTENSIONS`
|
||||
|
||||
- :setting:`FEED_EXPORTERS`
|
||||
|
||||
- :setting:`FEED_STORAGES`
|
||||
|
||||
- :setting:`ITEM_PIPELINES`
|
||||
|
||||
- :setting:`SCHEDULER`
|
||||
|
||||
- :setting:`SCHEDULER_DISK_QUEUE`
|
||||
|
||||
- :setting:`SCHEDULER_MEMORY_QUEUE`
|
||||
|
||||
- :setting:`SCHEDULER_PRIORITY_QUEUE`
|
||||
|
||||
- :setting:`SCHEDULER_START_DISK_QUEUE`
|
||||
|
||||
- :setting:`SCHEDULER_START_MEMORY_QUEUE`
|
||||
|
||||
- :setting:`SPIDER_MIDDLEWARES`
|
||||
- :setting:`ADDONS`
|
||||
- :setting:`DOWNLOAD_HANDLERS`
|
||||
- :setting:`DOWNLOADER_MIDDLEWARES`
|
||||
- :setting:`DUPEFILTER_CLASS`
|
||||
- :setting:`EXTENSIONS`
|
||||
- :setting:`FEED_EXPORTERS`
|
||||
- :setting:`FEED_STORAGES`
|
||||
- :setting:`ITEM_PIPELINES`
|
||||
- :setting:`SCHEDULER`
|
||||
- :setting:`SCHEDULER_DISK_QUEUE`
|
||||
- :setting:`SCHEDULER_MEMORY_QUEUE`
|
||||
- :setting:`SCHEDULER_PRIORITY_QUEUE`
|
||||
- :setting:`SCHEDULER_START_DISK_QUEUE`
|
||||
- :setting:`SCHEDULER_START_MEMORY_QUEUE`
|
||||
- :setting:`SPIDER_MIDDLEWARES`
|
||||
- :setting:`TWISTED_DNS_RESOLVER`
|
||||
|
||||
Third-party Scrapy components may also let you define additional Scrapy
|
||||
components, usually configurable through :ref:`settings <topics-settings>`, to
|
||||
|
|
|
|||
|
|
@ -136,18 +136,10 @@ Core Stats extension
|
|||
Enable the collection of core statistics, provided the stats collection is
|
||||
enabled (see :ref:`topics-stats`).
|
||||
|
||||
The following stats are collected:
|
||||
|
||||
* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`).
|
||||
* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`).
|
||||
* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`).
|
||||
* ``finish_reason``: the closing reason string (e.g. ``"finished"``,
|
||||
``"closespider_timeout"``).
|
||||
* ``item_scraped_count``: total number of items that passed all pipelines.
|
||||
* ``item_dropped_count``: total number of items dropped by a pipeline.
|
||||
* ``item_dropped_reasons_count/<ExceptionName>``: per-exception drop count
|
||||
(e.g. ``item_dropped_reasons_count/DropItem``).
|
||||
* ``response_received_count``: total number of HTTP responses received.
|
||||
The following stats are collected: :stat:`elapsed_time_seconds`,
|
||||
:stat:`finish_reason`, :stat:`finish_time`, :stat:`item_dropped_count`,
|
||||
:stat:`item_dropped_reasons_count/{exception}`, :stat:`item_scraped_count`,
|
||||
:stat:`response_received_count`, :stat:`start_time`.
|
||||
|
||||
Log Count extension
|
||||
~~~~~~~~~~~~~~~~~~~
|
||||
|
|
@ -190,7 +182,7 @@ Monitors the memory used by the Scrapy process that runs the spider and:
|
|||
|
||||
1. sends a :signal:`memusage_warning_reached` signal when it exceeds
|
||||
:setting:`MEMUSAGE_WARNING_MB`
|
||||
2. closes the spider with the `"memusage_exceeded"` reason when it exceeds
|
||||
2. closes the spider with the ``"memusage_exceeded"`` reason when it exceeds
|
||||
:setting:`MEMUSAGE_LIMIT_MB`
|
||||
|
||||
This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and
|
||||
|
|
@ -214,7 +206,8 @@ An extension for debugging memory usage. It collects information about:
|
|||
* objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs`
|
||||
|
||||
To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The
|
||||
info will be stored in the stats.
|
||||
info will be stored in the :stat:`memdebug/gc_garbage_count` and
|
||||
:stat:`memdebug/live_refs/{cls}` stats.
|
||||
|
||||
.. _topics-extensions-ref-spiderstate:
|
||||
|
||||
|
|
|
|||
|
|
@ -83,6 +83,14 @@ stopping it cleanly. Forced, sudden or otherwise unclean shutdown can lead to
|
|||
data corruption in the job directory, which may prevent the spider from
|
||||
resuming correctly.
|
||||
|
||||
Scrapy version changes
|
||||
----------------------
|
||||
|
||||
The contents of a job directory are an implementation detail of the Scrapy
|
||||
version that wrote them. A job must be resumed with the same Scrapy version
|
||||
that paused it; after upgrading or downgrading Scrapy, start a new job with a
|
||||
new job directory.
|
||||
|
||||
Cookies expiration
|
||||
------------------
|
||||
|
||||
|
|
|
|||
|
|
@ -575,7 +575,7 @@ CONCURRENT_REQUESTS
|
|||
Default: ``16``
|
||||
|
||||
The maximum number of concurrent (i.e. simultaneous) requests that will be
|
||||
performed by the Scrapy downloader.
|
||||
performed by the Scrapy downloader. Use ``0`` for no limit.
|
||||
|
||||
.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
|
||||
|
||||
|
|
@ -1872,7 +1872,8 @@ Default: ``False``
|
|||
|
||||
Setting to ``True`` will log debug information about the requests scheduler.
|
||||
This currently logs (only once) if the requests cannot be serialized to disk.
|
||||
Stats counter (``scheduler/unserializable``) tracks the number of times this happens.
|
||||
The :stat:`scheduler/unserializable` stat tracks the number of times this
|
||||
happens.
|
||||
|
||||
Example entry in logs::
|
||||
|
||||
|
|
|
|||
|
|
@ -21,6 +21,8 @@ using the Stats Collector from.
|
|||
Another feature of the Stats Collector is that it's very efficient (when
|
||||
enabled) and extremely efficient (almost unnoticeable) when disabled.
|
||||
|
||||
See :ref:`topics-stats-reference` below for the stats that Scrapy sets.
|
||||
|
||||
.. _topics-stats-usecases:
|
||||
|
||||
Common Stats Collector uses
|
||||
|
|
@ -101,3 +103,642 @@ DummyStatsCollector
|
|||
-------------------
|
||||
|
||||
.. autoclass:: DummyStatsCollector
|
||||
|
||||
.. _topics-stats-reference:
|
||||
|
||||
Built-in stats reference
|
||||
========================
|
||||
|
||||
Scrapy sets the following :ref:`stats <topics-stats>`. Components other than
|
||||
those built into Scrapy may set additional stats; see their documentation.
|
||||
|
||||
Stat keys that contain a ``{placeholder}`` below stand for a family of stats,
|
||||
one per actual value of the placeholder.
|
||||
|
||||
.. note:: Most stats are set by a specific :ref:`component
|
||||
<topics-components>`, and are only present if that component is enabled and
|
||||
its code path is reached. A stat that is missing from
|
||||
:meth:`~scrapy.statscollectors.StatsCollector.get_stats` output is
|
||||
equivalent to a counter of 0.
|
||||
|
||||
.. stat:: downloader/exception_count
|
||||
|
||||
``downloader/exception_count``
|
||||
Number of exceptions raised while downloading requests.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
||||
|
||||
.. stat:: downloader/exception_type_count/{exception_type}
|
||||
|
||||
``downloader/exception_type_count/{exception_type}``
|
||||
Number of exceptions raised while downloading requests, per exception type,
|
||||
where ``{exception_type}`` is the import path of the exception class, e.g.
|
||||
``twisted.internet.error.DNSLookupError``.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
||||
|
||||
.. stat:: downloader/request_bytes
|
||||
|
||||
``downloader/request_bytes``
|
||||
Total size, in bytes, of the requests sent, counting the request line, the
|
||||
headers and the body. As with :stat:`downloader/request_count`, requests
|
||||
served from the cache are also counted.
|
||||
|
||||
It is an approximation, reconstructed from each :class:`~scrapy.Request`
|
||||
object instead of measured on the wire, so it does not account for the
|
||||
actual bytes that the :ref:`download handler
|
||||
<topics-download-handlers>` sends, e.g. transport-level overhead.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
||||
|
||||
.. stat:: downloader/request_count
|
||||
|
||||
``downloader/request_count``
|
||||
Number of requests sent.
|
||||
|
||||
Requests that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
|
||||
serves from the cache are also counted, even though they are never sent,
|
||||
because it handles requests after
|
||||
:class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
||||
|
||||
.. stat:: downloader/request_method_count/{method}
|
||||
|
||||
``downloader/request_method_count/{method}``
|
||||
Number of requests sent, per HTTP method, e.g. ``GET`` or ``POST``. As with
|
||||
:stat:`downloader/request_count`, requests served from the cache are also
|
||||
counted.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
||||
|
||||
.. stat:: downloader/response_bytes
|
||||
|
||||
``downloader/response_bytes``
|
||||
Total size, in bytes, of the responses received, counting the status line,
|
||||
the headers and the body. It covers the same responses as
|
||||
:stat:`downloader/response_count`.
|
||||
|
||||
The body is counted as received, i.e. still compressed for responses that
|
||||
used ``Content-Encoding``, because
|
||||
:class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` handles
|
||||
responses before
|
||||
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
|
||||
decompresses them. See :stat:`httpcompression/response_bytes` for
|
||||
decompressed sizes.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
||||
|
||||
.. stat:: downloader/response_count
|
||||
|
||||
``downloader/response_count``
|
||||
Number of responses received.
|
||||
|
||||
It counts responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
|
||||
serves from the cache, even though they do not come from the network, and
|
||||
responses that a downloader middleware consumes before they reach your
|
||||
spider, e.g. redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware`
|
||||
turns into new requests. Compare with :stat:`response_received_count`.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
||||
|
||||
.. stat:: downloader/response_status_count/{status_code}
|
||||
|
||||
``downloader/response_status_count/{status_code}``
|
||||
Number of responses received, per HTTP status code, e.g. ``200`` or
|
||||
``404``. It covers the same responses as :stat:`downloader/response_count`.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
|
||||
|
||||
.. stat:: dupefilter/filtered
|
||||
|
||||
``dupefilter/filtered``
|
||||
Number of requests dropped as duplicates.
|
||||
|
||||
Set by :class:`~scrapy.dupefilters.RFPDupeFilter`.
|
||||
|
||||
.. stat:: elapsed_time_seconds
|
||||
|
||||
``elapsed_time_seconds``
|
||||
Time, as a :class:`float`, in seconds, between the :signal:`spider_opened`
|
||||
and the :signal:`spider_closed` signals.
|
||||
|
||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
||||
|
||||
.. stat:: feedexport/failed_count/{storage}
|
||||
|
||||
``feedexport/failed_count/{storage}``
|
||||
Number of :ref:`feeds <topics-feed-exports>` that could not be stored, per
|
||||
:ref:`storage backend <topics-feed-storage-backends>`, where ``{storage}``
|
||||
is the class name of the storage backend, e.g. ``FileFeedStorage``.
|
||||
|
||||
.. stat:: feedexport/success_count/{storage}
|
||||
|
||||
``feedexport/success_count/{storage}``
|
||||
Number of :ref:`feeds <topics-feed-exports>` stored successfully, per
|
||||
:ref:`storage backend <topics-feed-storage-backends>`, where ``{storage}``
|
||||
is the class name of the storage backend, e.g. ``FileFeedStorage``.
|
||||
|
||||
.. stat:: file_count
|
||||
|
||||
``file_count``
|
||||
Number of files handled by the :ref:`media pipelines
|
||||
<topics-media-pipeline>`.
|
||||
|
||||
.. stat:: file_status_count/{status}
|
||||
|
||||
``file_status_count/{status}``
|
||||
Number of files handled by the :ref:`media pipelines
|
||||
<topics-media-pipeline>`, per status, where ``{status}`` is one of:
|
||||
|
||||
- ``downloaded``: the file was downloaded.
|
||||
|
||||
- ``cached``: the file came from the
|
||||
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
|
||||
cache.
|
||||
|
||||
- ``uptodate``: the file was already in the storage backend and had not
|
||||
:ref:`expired <file-expiration>`, so it was not downloaded again.
|
||||
|
||||
.. stat:: finish_reason
|
||||
|
||||
``finish_reason``
|
||||
String indicating why the crawl finished. It matches the *reason* argument
|
||||
of the :signal:`spider_closed` signal.
|
||||
|
||||
Scrapy uses the following reasons:
|
||||
|
||||
- ``cancelled``: the spider was closed without a more specific reason,
|
||||
e.g. because :exc:`~scrapy.exceptions.CloseSpider` was raised without
|
||||
one.
|
||||
|
||||
- ``closespider_errorcount``: see :setting:`CLOSESPIDER_ERRORCOUNT`.
|
||||
|
||||
- ``closespider_itemcount``: see :setting:`CLOSESPIDER_ITEMCOUNT`.
|
||||
|
||||
- ``closespider_pagecount``: see :setting:`CLOSESPIDER_PAGECOUNT`.
|
||||
|
||||
- ``closespider_pagecount_no_item``: see
|
||||
:setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM`.
|
||||
|
||||
- ``closespider_timeout``: see :setting:`CLOSESPIDER_TIMEOUT`.
|
||||
|
||||
- ``closespider_timeout_no_item``: see
|
||||
:setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`.
|
||||
|
||||
- ``finished``: the spider became idle with no pending requests, i.e. it
|
||||
finished normally.
|
||||
|
||||
- ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`.
|
||||
|
||||
- ``shutdown``: the crawl was interrupted, e.g. by a system signal such
|
||||
as ``SIGINT`` (:kbd:`Ctrl-C`).
|
||||
|
||||
Third-party components and your own code may use any other reason, e.g. by
|
||||
raising :exc:`~scrapy.exceptions.CloseSpider` with it.
|
||||
|
||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
||||
|
||||
.. stat:: finish_time
|
||||
|
||||
``finish_time``
|
||||
Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when
|
||||
the :signal:`spider_closed` signal was sent.
|
||||
|
||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
||||
|
||||
.. stat:: httpcache/errorrecovery
|
||||
|
||||
``httpcache/errorrecovery``
|
||||
Number of times that a stale cached response was used because downloading a
|
||||
fresh response raised an exception.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
||||
|
||||
.. stat:: httpcache/firsthand
|
||||
|
||||
``httpcache/firsthand``
|
||||
Number of responses that were downloaded without a matching cache entry to
|
||||
validate against, i.e. responses for requests counted in
|
||||
:stat:`httpcache/miss`.
|
||||
|
||||
It is lower than :stat:`httpcache/miss` when some of those requests yield
|
||||
no response, either because they are dropped (see
|
||||
:stat:`httpcache/ignore`) or because their download fails.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
||||
|
||||
.. stat:: httpcache/hit
|
||||
|
||||
``httpcache/hit``
|
||||
Number of requests served from the cache.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
||||
|
||||
.. stat:: httpcache/ignore
|
||||
|
||||
``httpcache/ignore``
|
||||
Number of requests dropped because they were not in the cache and
|
||||
:setting:`HTTPCACHE_IGNORE_MISSING` is ``True``.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
||||
|
||||
.. stat:: httpcache/invalidate
|
||||
|
||||
``httpcache/invalidate``
|
||||
Number of times that a cached response failed validation and was replaced
|
||||
with a freshly downloaded response.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
||||
|
||||
.. stat:: httpcache/miss
|
||||
|
||||
``httpcache/miss``
|
||||
Number of requests for which no cache entry could be read, either because
|
||||
there was none or because reading it failed, in which case the request is
|
||||
also counted in :stat:`httpcache/retrieve_error`. Those requests are
|
||||
downloaded (see :stat:`httpcache/firsthand`), or dropped if
|
||||
:setting:`HTTPCACHE_IGNORE_MISSING` is ``True`` (see
|
||||
:stat:`httpcache/ignore`).
|
||||
|
||||
Requests with a stale cache entry are not counted here; see
|
||||
:stat:`httpcache/revalidate` and :stat:`httpcache/invalidate`.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
||||
|
||||
.. stat:: httpcache/retrieve_error
|
||||
|
||||
``httpcache/retrieve_error``
|
||||
Number of cache entries that could not be read, and hence were treated as
|
||||
cache misses. Those requests are also counted in :stat:`httpcache/miss`.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
||||
|
||||
.. stat:: httpcache/revalidate
|
||||
|
||||
``httpcache/revalidate``
|
||||
Number of times that a cached response was successfully validated against
|
||||
the target server, and hence used instead of the fresh response.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
||||
|
||||
.. stat:: httpcache/store
|
||||
|
||||
``httpcache/store``
|
||||
Number of responses stored in the cache.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
||||
|
||||
.. stat:: httpcache/uncacheable
|
||||
|
||||
``httpcache/uncacheable``
|
||||
Number of responses not stored in the cache because the
|
||||
:setting:`HTTPCACHE_POLICY` did not allow it.
|
||||
|
||||
Every response considered for caching is counted either here or in
|
||||
:stat:`httpcache/store`, so ``httpcache/store + httpcache/uncacheable``
|
||||
equals ``httpcache/firsthand + httpcache/invalidate``.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
|
||||
|
||||
.. stat:: httpcompression/response_bytes
|
||||
|
||||
``httpcompression/response_bytes``
|
||||
Total size, in bytes, of decompressed response bodies, counting only the
|
||||
body and only responses that were actually decompressed. Compare with
|
||||
:stat:`downloader/response_bytes`.
|
||||
|
||||
Set by
|
||||
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`.
|
||||
|
||||
.. stat:: httpcompression/response_count
|
||||
|
||||
``httpcompression/response_count``
|
||||
Number of decompressed responses.
|
||||
|
||||
Set by
|
||||
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`.
|
||||
|
||||
.. stat:: httperror/response_ignored_count
|
||||
|
||||
``httperror/response_ignored_count``
|
||||
Number of responses dropped because of their HTTP status code.
|
||||
|
||||
Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`.
|
||||
|
||||
.. stat:: httperror/response_ignored_status_count/{status_code}
|
||||
|
||||
``httperror/response_ignored_status_count/{status_code}``
|
||||
Number of responses dropped because of their HTTP status code, per HTTP
|
||||
status code, e.g. ``404``.
|
||||
|
||||
Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`.
|
||||
|
||||
.. stat:: item_dropped_count
|
||||
|
||||
``item_dropped_count``
|
||||
Number of items dropped by an :ref:`item pipeline
|
||||
<topics-item-pipeline>`, i.e. number of times that the
|
||||
:signal:`item_dropped` signal was sent.
|
||||
|
||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
||||
|
||||
.. stat:: item_dropped_reasons_count/{exception}
|
||||
|
||||
``item_dropped_reasons_count/{exception}``
|
||||
Number of items dropped, per exception, where ``{exception}`` is the class
|
||||
name of the exception that caused the item to be dropped.
|
||||
|
||||
Only :exc:`~scrapy.exceptions.DropItem` and its subclasses drop items, and
|
||||
each one is counted under its own class name, e.g.
|
||||
``item_dropped_reasons_count/DropItem`` for
|
||||
:exc:`~scrapy.exceptions.DropItem` itself and
|
||||
``item_dropped_reasons_count/MyDropItem`` for a ``MyDropItem`` subclass of
|
||||
it. Any other exception raised by an :ref:`item pipeline
|
||||
<topics-item-pipeline>` triggers the :signal:`item_error` signal instead of
|
||||
:signal:`item_dropped`, and is not counted here or in
|
||||
:stat:`item_dropped_count`.
|
||||
|
||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
||||
|
||||
.. stat:: item_scraped_count
|
||||
|
||||
``item_scraped_count``
|
||||
Number of items that passed all :ref:`item pipelines
|
||||
<topics-item-pipeline>`, i.e. number of times that the
|
||||
:signal:`item_scraped` signal was sent.
|
||||
|
||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
||||
|
||||
.. stat:: items_per_minute
|
||||
|
||||
``items_per_minute``
|
||||
Average number of items scraped per minute during the crawl.
|
||||
|
||||
It is ``None`` if the crawl took less than a minute.
|
||||
|
||||
Set by :class:`~scrapy.extensions.logstats.LogStats`.
|
||||
|
||||
.. stat:: log_count/{level}
|
||||
|
||||
``log_count/{level}``
|
||||
Number of log messages, per logging level name, e.g. ``INFO`` or
|
||||
``WARNING``.
|
||||
|
||||
Only messages that the :setting:`LOG_LEVEL` setting allows are counted.
|
||||
|
||||
Set by :class:`~scrapy.extensions.logcount.LogCount`.
|
||||
|
||||
.. stat:: memdebug/gc_garbage_count
|
||||
|
||||
``memdebug/gc_garbage_count``
|
||||
Number of objects in :data:`gc.garbage` when the spider is closed.
|
||||
|
||||
Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires
|
||||
:setting:`MEMDEBUG_ENABLED` to be ``True``.
|
||||
|
||||
.. stat:: memdebug/live_refs/{cls}
|
||||
|
||||
``memdebug/live_refs/{cls}``
|
||||
Number of live objects of class ``{cls}`` when the spider is closed, as
|
||||
reported by :ref:`trackref <topics-leaks-trackrefs>`, e.g.
|
||||
``memdebug/live_refs/HtmlResponse``.
|
||||
|
||||
Only set for classes with at least 1 live object.
|
||||
|
||||
Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires
|
||||
:setting:`MEMDEBUG_ENABLED` to be ``True``.
|
||||
|
||||
.. stat:: memusage/limit_reached
|
||||
|
||||
``memusage/limit_reached``
|
||||
``1`` if memory usage exceeded :setting:`MEMUSAGE_LIMIT_MB`, which also
|
||||
stops the crawl.
|
||||
|
||||
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
|
||||
|
||||
.. stat:: memusage/max
|
||||
|
||||
``memusage/max``
|
||||
Maximum peak memory usage, in bytes, observed during the crawl.
|
||||
|
||||
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
|
||||
|
||||
.. stat:: memusage/startup
|
||||
|
||||
``memusage/startup``
|
||||
Peak memory usage, in bytes, when the engine started.
|
||||
|
||||
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
|
||||
|
||||
.. stat:: memusage/warning_reached
|
||||
|
||||
``memusage/warning_reached``
|
||||
``1`` if memory usage exceeded :setting:`MEMUSAGE_WARNING_MB`.
|
||||
|
||||
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
|
||||
|
||||
.. stat:: offsite/domains
|
||||
|
||||
``offsite/domains``
|
||||
Number of distinct domains for which at least 1 request was dropped for
|
||||
being offsite.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`.
|
||||
|
||||
.. stat:: offsite/filtered
|
||||
|
||||
``offsite/filtered``
|
||||
Number of requests dropped for being offsite.
|
||||
|
||||
Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`.
|
||||
|
||||
.. stat:: request_depth_count/{depth}
|
||||
|
||||
``request_depth_count/{depth}``
|
||||
Number of requests scheduled at depth ``{depth}``, e.g.
|
||||
``request_depth_count/2``.
|
||||
|
||||
Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`, which
|
||||
requires :setting:`DEPTH_STATS_VERBOSE` to be ``True`` for this stat.
|
||||
|
||||
.. stat:: request_depth_max
|
||||
|
||||
``request_depth_max``
|
||||
Maximum depth reached.
|
||||
|
||||
Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`.
|
||||
|
||||
.. stat:: response_received_count
|
||||
|
||||
``response_received_count``
|
||||
Number of responses received, i.e. number of times that the
|
||||
:signal:`response_received` signal was sent.
|
||||
|
||||
Unlike :stat:`downloader/response_count`, it does not count responses that
|
||||
a downloader middleware consumes before they reach the engine, e.g.
|
||||
redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware`
|
||||
turns into new requests. Both count responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
|
||||
serves from the cache.
|
||||
|
||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
||||
|
||||
.. stat:: responses_per_minute
|
||||
|
||||
``responses_per_minute``
|
||||
Average number of responses received per minute during the crawl.
|
||||
|
||||
It is ``None`` if the crawl took less than a minute.
|
||||
|
||||
Set by :class:`~scrapy.extensions.logstats.LogStats`.
|
||||
|
||||
.. stat:: retry/count
|
||||
|
||||
``retry/count``
|
||||
Number of requests retried.
|
||||
|
||||
Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which
|
||||
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses.
|
||||
|
||||
.. stat:: retry/max_reached
|
||||
|
||||
``retry/max_reached``
|
||||
Number of requests that were not retried because they had already been
|
||||
retried :setting:`RETRY_TIMES` times.
|
||||
|
||||
Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which
|
||||
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses.
|
||||
|
||||
.. stat:: retry/reason_count/{reason}
|
||||
|
||||
``retry/reason_count/{reason}``
|
||||
Number of requests retried, per reason, e.g.
|
||||
``retry/reason_count/twisted.internet.error.TimeoutError`` or
|
||||
``retry/reason_count/504 Gateway Time-out``.
|
||||
|
||||
Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which
|
||||
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses.
|
||||
|
||||
.. note:: Code calling
|
||||
:func:`~scrapy.downloadermiddlewares.retry.get_retry_request` may pass a
|
||||
custom *stats_base_key*, in which case ``retry`` is replaced with that key
|
||||
in the 3 stats above.
|
||||
|
||||
.. stat:: robotstxt/exception_count/{exception_type}
|
||||
|
||||
``robotstxt/exception_count/{exception_type}``
|
||||
Number of exceptions raised while downloading ``robots.txt`` files, per
|
||||
exception type, where ``{exception_type}`` is the string representation of
|
||||
the exception class, e.g. ``<class
|
||||
'twisted.internet.error.DNSLookupError'>``.
|
||||
|
||||
Set by
|
||||
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
|
||||
|
||||
.. stat:: robotstxt/forbidden
|
||||
|
||||
``robotstxt/forbidden``
|
||||
Number of requests dropped for being disallowed by ``robots.txt``.
|
||||
|
||||
Set by
|
||||
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
|
||||
|
||||
.. stat:: robotstxt/request_count
|
||||
|
||||
``robotstxt/request_count``
|
||||
Number of ``robots.txt`` files requested, i.e. 1 per network location for
|
||||
which at least 1 request was sent.
|
||||
|
||||
Set by
|
||||
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
|
||||
|
||||
.. stat:: robotstxt/response_count
|
||||
|
||||
``robotstxt/response_count``
|
||||
Number of ``robots.txt`` responses received.
|
||||
|
||||
Set by
|
||||
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
|
||||
|
||||
.. stat:: robotstxt/response_status_count/{status_code}
|
||||
|
||||
``robotstxt/response_status_count/{status_code}``
|
||||
Number of ``robots.txt`` responses received, per HTTP status code, e.g.
|
||||
``404``.
|
||||
|
||||
Set by
|
||||
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
|
||||
|
||||
.. stat:: scheduler/dequeued
|
||||
|
||||
``scheduler/dequeued``
|
||||
Number of requests read from the :ref:`scheduler <topics-scheduler>`.
|
||||
|
||||
.. stat:: scheduler/dequeued/disk
|
||||
|
||||
``scheduler/dequeued/disk``
|
||||
Number of requests read from the disk queue of the :ref:`scheduler
|
||||
<topics-scheduler>`.
|
||||
|
||||
.. stat:: scheduler/dequeued/memory
|
||||
|
||||
``scheduler/dequeued/memory``
|
||||
Number of requests read from the memory queue of the :ref:`scheduler
|
||||
<topics-scheduler>`.
|
||||
|
||||
.. stat:: scheduler/enqueued
|
||||
|
||||
``scheduler/enqueued``
|
||||
Number of requests stored into the :ref:`scheduler <topics-scheduler>`.
|
||||
|
||||
.. stat:: scheduler/enqueued/disk
|
||||
|
||||
``scheduler/enqueued/disk``
|
||||
Number of requests stored into the disk queue of the :ref:`scheduler
|
||||
<topics-scheduler>`.
|
||||
|
||||
.. stat:: scheduler/enqueued/memory
|
||||
|
||||
``scheduler/enqueued/memory``
|
||||
Number of requests stored into the memory queue of the :ref:`scheduler
|
||||
<topics-scheduler>`.
|
||||
|
||||
.. stat:: scheduler/unserializable
|
||||
|
||||
``scheduler/unserializable``
|
||||
Number of requests that could not be stored into the disk queue of the
|
||||
:ref:`scheduler <topics-scheduler>` because they could not be
|
||||
:ref:`serialized <request-serialization>`, and hence were stored into the
|
||||
memory queue instead.
|
||||
|
||||
.. stat:: spider_exceptions/count
|
||||
|
||||
``spider_exceptions/count``
|
||||
Number of unhandled exceptions raised by spider callbacks.
|
||||
|
||||
Set by the :ref:`scraper <topics-architecture>`.
|
||||
|
||||
.. stat:: spider_exceptions/{exception}
|
||||
|
||||
``spider_exceptions/{exception}``
|
||||
Number of unhandled exceptions raised by spider callbacks, per exception,
|
||||
where ``{exception}`` is the class name of the exception, e.g.
|
||||
``spider_exceptions/ValueError``.
|
||||
|
||||
Set by the :ref:`scraper <topics-architecture>`.
|
||||
|
||||
.. stat:: start_time
|
||||
|
||||
``start_time``
|
||||
Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when
|
||||
the :signal:`spider_opened` signal was sent.
|
||||
|
||||
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
|
||||
|
||||
.. stat:: urllength/request_ignored_count
|
||||
|
||||
``urllength/request_ignored_count``
|
||||
Number of requests dropped for having a URL longer than
|
||||
:setting:`URLLENGTH_LIMIT`.
|
||||
|
||||
Set by :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`.
|
||||
|
|
|
|||
|
|
@ -71,8 +71,8 @@ brotli = [
|
|||
gcs = ["google-cloud-storage>=1.29.0"]
|
||||
httpx = ["httpx2[http2,socks]>=2.0.0"]
|
||||
images = ["Pillow>=8.3.2"]
|
||||
ipython = ["ipython>=7.1.0"]
|
||||
ptpython = ["ptpython>=2.0.1"]
|
||||
ipython = ["ipython>=8.15.0"]
|
||||
ptpython = ["ptpython>=3.0.23"]
|
||||
robotparser = ["robotexclusionrulesparser>=1.6.2"]
|
||||
s3 = ["boto3>=1.20.0"]
|
||||
twisted-http2 = ["Twisted[http2]>=21.7.0"]
|
||||
|
|
@ -127,17 +127,6 @@ module = [
|
|||
"tests.test_contracts",
|
||||
"tests.test_core_downloader",
|
||||
"tests.test_downloader_handler_twisted_ftp",
|
||||
"tests.test_downloadermiddleware_cookies",
|
||||
"tests.test_downloadermiddleware_httpauth",
|
||||
"tests.test_downloadermiddleware_httpcache",
|
||||
"tests.test_downloadermiddleware_httpcompression",
|
||||
"tests.test_downloadermiddleware_httpproxy",
|
||||
"tests.test_downloadermiddleware_offsite",
|
||||
"tests.test_downloadermiddleware_redirect",
|
||||
"tests.test_downloadermiddleware_redirect_base",
|
||||
"tests.test_downloadermiddleware_redirect_metarefresh",
|
||||
"tests.test_downloadermiddleware_retry",
|
||||
"tests.test_downloadermiddleware_robotstxt",
|
||||
"tests.test_downloaderslotssettings",
|
||||
"tests.test_dupefilters",
|
||||
"tests.test_engine_loop",
|
||||
|
|
@ -149,11 +138,6 @@ module = [
|
|||
"tests.test_feedexport_storages",
|
||||
"tests.test_feedexport_uri_params",
|
||||
"tests.test_http2_client_protocol",
|
||||
"tests.test_http_headers",
|
||||
"tests.test_http_request",
|
||||
"tests.test_http_request_form",
|
||||
"tests.test_http_response",
|
||||
"tests.test_http_response_text",
|
||||
"tests.test_item",
|
||||
"tests.test_linkextractors",
|
||||
"tests.test_loader",
|
||||
|
|
@ -165,11 +149,6 @@ module = [
|
|||
"tests.test_pipeline_media",
|
||||
"tests.test_pipelines",
|
||||
"tests.test_pqueues",
|
||||
"tests.test_request_attribute_binding",
|
||||
"tests.test_request_cb_kwargs",
|
||||
"tests.test_request_dict",
|
||||
"tests.test_request_left",
|
||||
"tests.test_robotstxt_interface",
|
||||
"tests.test_scheduler_base",
|
||||
"tests.test_settings",
|
||||
"tests.test_spider",
|
||||
|
|
@ -180,8 +159,6 @@ module = [
|
|||
"tests.test_squeues",
|
||||
"tests.test_squeues_request",
|
||||
"tests.test_stats",
|
||||
"tests.utils.bases.http_request",
|
||||
"tests.utils.bases.http_response",
|
||||
"tests.utils.bases.spider",
|
||||
]
|
||||
check_untyped_defs = false
|
||||
|
|
|
|||
|
|
@ -138,7 +138,8 @@ class Downloader:
|
|||
self.active.remove(request)
|
||||
|
||||
def needs_backout(self) -> bool:
|
||||
return len(self.active) >= self.total_concurrency
|
||||
# A total concurrency of 0 means no limit.
|
||||
return 0 < self.total_concurrency <= len(self.active)
|
||||
|
||||
@_warn_spider_arg
|
||||
def _get_slot(
|
||||
|
|
|
|||
|
|
@ -92,10 +92,11 @@ class HttpxDownloadHandler(_Base):
|
|||
self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
|
||||
self._bind_host: str | None = self._get_bind_address_host()
|
||||
self._limits: httpx.Limits = httpx.Limits(
|
||||
# hard limit on simultaneous connections
|
||||
max_connections=self._pool_size_total,
|
||||
# hard limit on simultaneous connections (None for no limit, which
|
||||
# is what a CONCURRENT_REQUESTS of 0 means)
|
||||
max_connections=self._pool_size_total or None,
|
||||
# total number of idle connections in the pool (extra ones are closed)
|
||||
max_keepalive_connections=self._pool_size_total,
|
||||
max_keepalive_connections=self._pool_size_total or None,
|
||||
)
|
||||
|
||||
self._default_client: httpx.AsyncClient = self._make_client()
|
||||
|
|
|
|||
|
|
@ -366,8 +366,8 @@ class Scheduler(BaseScheduler):
|
|||
Unless the received request is filtered out by the Dupefilter, attempt to push
|
||||
it into the disk queue, falling back to pushing it into the memory queue.
|
||||
|
||||
Increment the appropriate stats, such as: ``scheduler/enqueued``,
|
||||
``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``.
|
||||
Increment the appropriate stats, such as: :stat:`scheduler/enqueued`,
|
||||
:stat:`scheduler/enqueued/disk`, :stat:`scheduler/enqueued/memory`.
|
||||
|
||||
Return ``True`` if the request was stored successfully, ``False`` otherwise.
|
||||
"""
|
||||
|
|
@ -390,8 +390,8 @@ class Scheduler(BaseScheduler):
|
|||
falling back to the disk queue if the memory queue is empty.
|
||||
Return ``None`` if there are no more enqueued requests.
|
||||
|
||||
Increment the appropriate stats, such as: ``scheduler/dequeued``,
|
||||
``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``.
|
||||
Increment the appropriate stats, such as: :stat:`scheduler/dequeued`,
|
||||
:stat:`scheduler/dequeued/disk`, :stat:`scheduler/dequeued/memory`.
|
||||
"""
|
||||
request: Request | None = self.mqs.pop()
|
||||
assert self.stats is not None
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ class LogCount:
|
|||
"""Install a log handler that counts log messages by level.
|
||||
|
||||
The handler installed is :class:`scrapy.utils.log.LogCounterHandler`.
|
||||
The counts are stored in stats as ``log_count/<level>``.
|
||||
The counts are stored in the :stat:`log_count/{level}` stat.
|
||||
|
||||
.. versionadded:: 2.14
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -50,7 +50,9 @@ class VerboseCookie(TypedDict):
|
|||
secure: NotRequired[bool]
|
||||
|
||||
|
||||
CookiesT: TypeAlias = dict[str | bytes, str | bytes] | list[VerboseCookie]
|
||||
CookiesT: TypeAlias = (
|
||||
dict[str | bytes, str | bytes | bool | float | int] | list[VerboseCookie]
|
||||
)
|
||||
|
||||
|
||||
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast
|
||||
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
|
||||
from warnings import warn
|
||||
|
|
@ -34,7 +34,7 @@ if TYPE_CHECKING:
|
|||
|
||||
FormdataVType: TypeAlias = str | Iterable[str]
|
||||
FormdataKVType: TypeAlias = tuple[str, FormdataVType]
|
||||
FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None
|
||||
FormdataType: TypeAlias = Mapping[str, FormdataVType] | Iterable[FormdataKVType] | None
|
||||
|
||||
|
||||
class FormRequest(Request):
|
||||
|
|
@ -100,7 +100,7 @@ class FormRequest(Request):
|
|||
super().__init__(*args, **kwargs)
|
||||
|
||||
if formdata:
|
||||
items = formdata.items() if isinstance(formdata, dict) else formdata
|
||||
items = formdata.items() if isinstance(formdata, Mapping) else formdata
|
||||
form_query_str = _urlencode(items, self.encoding)
|
||||
if self.method == "POST":
|
||||
self.headers.setdefault(
|
||||
|
|
@ -248,7 +248,7 @@ def _get_inputs(
|
|||
if clickable and clickable[0] not in formdata and clickable[0] is not None:
|
||||
values.append(clickable)
|
||||
|
||||
formdata_items = formdata.items() if isinstance(formdata, dict) else formdata
|
||||
formdata_items = formdata.items() if isinstance(formdata, Mapping) else formdata
|
||||
values.extend((k, v) for k, v in formdata_items if v is not None)
|
||||
return values
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
|
|||
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar
|
||||
|
||||
from twisted.internet.defer import Deferred
|
||||
from twisted.internet.task import LoopingCall
|
||||
from twisted.internet.task import LoopingCall, deferLater
|
||||
from twisted.internet.threads import deferToThread
|
||||
|
||||
from scrapy.utils.asyncgen import as_async_generator
|
||||
|
|
@ -293,6 +293,24 @@ class CallLaterResult:
|
|||
self._delayed_call = None
|
||||
|
||||
|
||||
async def sleep(seconds: float) -> None:
|
||||
"""Sleep for *seconds*.
|
||||
|
||||
.. versionadded:: VERSION
|
||||
|
||||
This uses either :func:`asyncio.sleep` or
|
||||
:func:`~twisted.internet.task.deferLater`, depending on whether asyncio
|
||||
support is available.
|
||||
"""
|
||||
if is_asyncio_available():
|
||||
await asyncio.sleep(seconds)
|
||||
return
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
await deferLater(reactor, seconds)
|
||||
|
||||
|
||||
async def run_in_thread(
|
||||
func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs
|
||||
) -> _T:
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import code
|
||||
from collections.abc import Callable
|
||||
from functools import wraps
|
||||
from functools import partial, wraps
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -16,16 +17,8 @@ def _embed_ipython_shell(
|
|||
namespace: dict[str, Any] | None = None, banner: str = ""
|
||||
) -> EmbedFuncT:
|
||||
"""Start an IPython Shell"""
|
||||
try:
|
||||
from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415
|
||||
from IPython.terminal.ipapp import load_default_config # noqa: PLC0415
|
||||
except ImportError:
|
||||
from IPython.frontend.terminal.embed import ( # type: ignore[import-not-found,no-redef] # noqa: T100,PLC0415
|
||||
InteractiveShellEmbed,
|
||||
)
|
||||
from IPython.frontend.terminal.ipapp import ( # type: ignore[import-not-found,no-redef] # noqa: PLC0415
|
||||
load_default_config,
|
||||
)
|
||||
from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415
|
||||
from IPython.terminal.ipapp import load_default_config # noqa: PLC0415
|
||||
|
||||
@wraps(_embed_ipython_shell)
|
||||
def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
|
||||
|
|
@ -38,6 +31,19 @@ def _embed_ipython_shell(
|
|||
shell = InteractiveShellEmbed.instance(
|
||||
banner1=banner, user_ns=namespace, config=config
|
||||
)
|
||||
# If an asyncio event loop is already running in this thread, e.g. when
|
||||
# inspect_response() is called from a spider callback while using the
|
||||
# asyncio reactor, prompt_toolkit cannot run its own event loop here, so
|
||||
# ask it to run the prompt in a separate thread instead. pt_app is None
|
||||
# when IPython falls back to its simple prompt, which needs no event loop.
|
||||
# See https://github.com/scrapy/scrapy/issues/5447
|
||||
if (pt_app := getattr(shell, "pt_app", None)) is not None:
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
pass
|
||||
else:
|
||||
pt_app.prompt = partial(pt_app.prompt, in_thread=True)
|
||||
shell()
|
||||
|
||||
return wrapper
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from twisted.internet.task import Cooperator
|
|||
from twisted.python import failure
|
||||
|
||||
from scrapy.exceptions import ScrapyDeprecationWarning
|
||||
from scrapy.utils.asyncio import is_asyncio_available
|
||||
from scrapy.utils.asyncio import is_asyncio_available, sleep
|
||||
from scrapy.utils.python import global_object_name
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -90,14 +90,7 @@ async def _defer_sleep_async() -> None:
|
|||
"""Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers
|
||||
before attending pending delayed calls, so do not set delay to zero.
|
||||
"""
|
||||
if is_asyncio_available():
|
||||
await asyncio.sleep(_DEFER_DELAY)
|
||||
else:
|
||||
from twisted.internet import reactor
|
||||
|
||||
d: Deferred[None] = Deferred()
|
||||
reactor.callLater(_DEFER_DELAY, d.callback, None)
|
||||
await d
|
||||
await sleep(_DEFER_DELAY)
|
||||
|
||||
|
||||
def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover
|
||||
|
|
|
|||
|
|
@ -50,8 +50,13 @@ class _Pipeline:
|
|||
return item
|
||||
|
||||
|
||||
def test_benchmark_crawl(benchmark: BenchmarkFixture, mockserver: MockServer) -> None:
|
||||
"""Crawl of a set of interlinked pages served over HTTP."""
|
||||
def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None:
|
||||
"""Per-request overhead of a crawl over HTTP.
|
||||
|
||||
The pages are small on purpose, so that the cost of parsing them stays
|
||||
negligible next to the cost of moving requests and responses through the
|
||||
engine, the middlewares and the download handler.
|
||||
"""
|
||||
query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"})
|
||||
url = mockserver.url(f"/follow?{query}")
|
||||
settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False}
|
||||
|
|
@ -14,6 +14,7 @@ from twisted.web import server, static
|
|||
from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
|
||||
from twisted.web.client import Response as TxResponse
|
||||
|
||||
from scrapy import Request
|
||||
from scrapy.core.downloader import Downloader, Slot, tls
|
||||
from scrapy.core.downloader.contextfactory import (
|
||||
_load_context_factory_from_settings,
|
||||
|
|
@ -296,6 +297,23 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
|
|||
await self._assert_factory_works(server_url, client_context_factory)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("concurrency", "active", "expected"),
|
||||
[
|
||||
(2, 1, False),
|
||||
(2, 2, True),
|
||||
(0, 0, False),
|
||||
(0, 2, False),
|
||||
],
|
||||
)
|
||||
def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:
|
||||
crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})
|
||||
downloader = Downloader(crawler)
|
||||
downloader.active = {Request(f"https://example.com/{i}") for i in range(active)}
|
||||
assert downloader.needs_backout() is expected
|
||||
downloader.close()
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_fetch_deprecated_spider_arg():
|
||||
class CustomDownloader(Downloader):
|
||||
|
|
|
|||
|
|
@ -14,7 +14,8 @@ from packaging.version import parse as parse_version
|
|||
from pexpect.popen_spawn import PopenSpawn
|
||||
from w3lib import __version__ as w3lib_version
|
||||
|
||||
from tests.utils import async_sleep, get_script_run_env
|
||||
from scrapy.utils.asyncio import sleep
|
||||
from tests.utils import get_script_run_env
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -244,7 +245,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
|
|||
p.kill(sig)
|
||||
p.expect_exact("shutting down gracefully")
|
||||
# sending the second signal too fast often causes problems
|
||||
await async_sleep(0.01)
|
||||
await sleep(0.01)
|
||||
p.kill(sig)
|
||||
p.expect_exact("forcing unclean shutdown")
|
||||
p.wait() # type: ignore[no-untyped-call]
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ from scrapy.core.downloader.handlers._httpx import (
|
|||
HttpxDownloadHandler,
|
||||
)
|
||||
from scrapy.exceptions import DownloadFailedError
|
||||
from scrapy.utils.misc import build_from_crawler
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.utils.bases.download_handlers_http import (
|
||||
TestHttpBase,
|
||||
TestHttpProxyBase,
|
||||
|
|
@ -161,3 +163,15 @@ class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
|
|||
@pytest.mark.requires_internet
|
||||
class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("concurrency", "expected"), [(16, 16), (0, None)])
|
||||
@coroutine_test
|
||||
async def test_pool_limits(concurrency: int, expected: int | None) -> None:
|
||||
crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})
|
||||
handler = build_from_crawler(HttpxDownloadHandler, crawler)
|
||||
try:
|
||||
assert handler._limits.max_connections == expected
|
||||
assert handler._limits.max_keepalive_connections == expected
|
||||
finally:
|
||||
await handler.close()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import logging
|
||||
from collections.abc import Iterable
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -219,7 +220,7 @@ class TestCookiesMiddleware:
|
|||
|
||||
def test_complex_cookies(self):
|
||||
# merge some cookies into jar
|
||||
cookies = [
|
||||
cookies: list[VerboseCookie] = [
|
||||
{
|
||||
"name": "C1",
|
||||
"value": "value1",
|
||||
|
|
@ -483,13 +484,13 @@ class TestCookiesMiddleware:
|
|||
|
||||
def _test_cookie_redirect(
|
||||
self,
|
||||
source,
|
||||
target,
|
||||
source: str | dict[str, Any],
|
||||
target: str | dict[str, Any],
|
||||
*,
|
||||
cookies1,
|
||||
cookies2,
|
||||
):
|
||||
input_cookies = {"a": "b"}
|
||||
cookies1: bool,
|
||||
cookies2: bool,
|
||||
) -> None:
|
||||
input_cookies: CookiesT = {"a": "b"}
|
||||
|
||||
if not isinstance(source, dict):
|
||||
source = {"url": source}
|
||||
|
|
@ -551,11 +552,11 @@ class TestCookiesMiddleware:
|
|||
|
||||
def _test_cookie_header_redirect(
|
||||
self,
|
||||
source,
|
||||
target,
|
||||
source: str | dict[str, Any],
|
||||
target: str | dict[str, Any],
|
||||
*,
|
||||
cookies2,
|
||||
):
|
||||
cookies2: bool,
|
||||
) -> None:
|
||||
"""Test the handling of a user-defined Cookie header when building a
|
||||
redirect follow-up request.
|
||||
|
||||
|
|
@ -623,14 +624,14 @@ class TestCookiesMiddleware:
|
|||
|
||||
def _test_user_set_cookie_domain_followup(
|
||||
self,
|
||||
url1,
|
||||
url2,
|
||||
domain,
|
||||
url1: str,
|
||||
url2: str,
|
||||
domain: str,
|
||||
*,
|
||||
cookies1,
|
||||
cookies2,
|
||||
):
|
||||
input_cookies = [
|
||||
cookies1: bool,
|
||||
cookies2: bool,
|
||||
) -> None:
|
||||
input_cookies: list[VerboseCookie] = [
|
||||
{
|
||||
"name": "a",
|
||||
"value": "b",
|
||||
|
|
@ -686,16 +687,16 @@ class TestCookiesMiddleware:
|
|||
|
||||
def _test_server_set_cookie_domain_followup(
|
||||
self,
|
||||
url1,
|
||||
url2,
|
||||
domain,
|
||||
url1: str,
|
||||
url2: str,
|
||||
domain: str,
|
||||
*,
|
||||
cookies,
|
||||
):
|
||||
cookies: bool,
|
||||
) -> None:
|
||||
request1 = Request(url1)
|
||||
self.mw.process_request(request1)
|
||||
|
||||
input_cookies = [
|
||||
input_cookies: list[VerboseCookie] = [
|
||||
{
|
||||
"name": "a",
|
||||
"value": "b",
|
||||
|
|
@ -747,8 +748,14 @@ class TestCookiesMiddleware:
|
|||
)
|
||||
|
||||
def _test_cookie_redirect_scheme_change(
|
||||
self, secure, from_scheme, to_scheme, cookies1, cookies2, cookies3
|
||||
):
|
||||
self,
|
||||
secure: bool | object,
|
||||
from_scheme: str,
|
||||
to_scheme: str,
|
||||
cookies1: bool,
|
||||
cookies2: bool,
|
||||
cookies3: bool,
|
||||
) -> None:
|
||||
"""When a redirect causes the URL scheme to change from *from_scheme*
|
||||
to *to_scheme*, while domain and port remain the same, and given a
|
||||
cookie on the initial request with its secure attribute set to
|
||||
|
|
@ -756,10 +763,11 @@ class TestCookiesMiddleware:
|
|||
initial request (*cookies1*), if it should be kept by the redirect
|
||||
middleware (*cookies2*), and if it should be present on the Cookie
|
||||
header in the redirected request (*cookie3*)."""
|
||||
cookie_kwargs = {}
|
||||
cookie: VerboseCookie = {"name": "a", "value": "b"}
|
||||
if secure is not UNSET:
|
||||
cookie_kwargs["secure"] = secure
|
||||
input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}]
|
||||
assert isinstance(secure, bool)
|
||||
cookie["secure"] = secure
|
||||
input_cookies = [cookie]
|
||||
|
||||
request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies)
|
||||
self.mw.process_request(request1)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from w3lib.http import basic_auth_header
|
||||
|
||||
|
|
@ -10,8 +12,10 @@ from scrapy.utils.test import get_crawler
|
|||
_DOMAIN_NOT_SET = object()
|
||||
|
||||
|
||||
def make_mw(user="", passwd="", domain=_DOMAIN_NOT_SET):
|
||||
settings: dict = {
|
||||
def make_mw(
|
||||
user: str = "", passwd: str = "", domain: str | object = _DOMAIN_NOT_SET
|
||||
) -> HttpAuthMiddleware:
|
||||
settings: dict[str, Any] = {
|
||||
"HTTPAUTH_USER": user,
|
||||
"HTTPAUTH_PASS": passwd,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -94,14 +94,14 @@ class TestBase:
|
|||
finally:
|
||||
mw.spider_closed(crawler.spider)
|
||||
|
||||
def assertEqualResponse(self, response1, response2):
|
||||
def assertEqualResponse(self, response1: Response, response2: Response) -> None:
|
||||
assert response1.url == response2.url
|
||||
assert response1.status == response2.status
|
||||
assert response1.headers == response2.headers
|
||||
assert response1.body == response2.body
|
||||
|
||||
|
||||
class StorageTestMixin:
|
||||
class StorageTestMixin(TestBase):
|
||||
"""Mixin containing storage-specific test methods."""
|
||||
|
||||
def _corrupt_cache_entry(
|
||||
|
|
@ -135,6 +135,8 @@ class StorageTestMixin:
|
|||
def test_corrupted_cache_entry_is_a_miss(self, caplog):
|
||||
with self._middleware() as mw:
|
||||
spider = mw.crawler.spider
|
||||
assert spider
|
||||
assert mw.crawler.stats
|
||||
mw.storage.store_response(spider, self.request, self.response)
|
||||
self._corrupt_cache_entry(mw.storage, spider, self.request)
|
||||
|
||||
|
|
@ -155,6 +157,8 @@ class StorageTestMixin:
|
|||
def test_corrupted_cache_entry_ignore_missing(self):
|
||||
with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
|
||||
spider = mw.crawler.spider
|
||||
assert spider
|
||||
assert mw.crawler.stats
|
||||
mw.storage.store_response(spider, self.request, self.response)
|
||||
self._corrupt_cache_entry(mw.storage, spider, self.request)
|
||||
|
||||
|
|
@ -180,7 +184,7 @@ class StorageTestMixin:
|
|||
self.assertEqualResponse(response, cached_response)
|
||||
|
||||
|
||||
class PolicyTestMixin:
|
||||
class PolicyTestMixin(TestBase):
|
||||
"""Mixin containing policy-specific test methods."""
|
||||
|
||||
def test_dont_cache(self):
|
||||
|
|
@ -302,6 +306,7 @@ class DummyPolicyTestMixin(PolicyTestMixin):
|
|||
assert mw.process_request(self.request) is None
|
||||
fresh_response = self.response.replace(body=b"new body")
|
||||
response = mw.process_response(self.request, fresh_response)
|
||||
assert isinstance(response, Response)
|
||||
self.assertEqualResponse(self.response, response)
|
||||
assert "cached" in response.flags
|
||||
assert mw.stats.get_value("httpcache/revalidate") == 1
|
||||
|
|
@ -313,12 +318,12 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
|
|||
@staticmethod
|
||||
def _process_requestresponse(
|
||||
mw: HttpCacheMiddleware, request: Request, response: Response | None
|
||||
) -> Response | Request:
|
||||
result = None
|
||||
) -> Response:
|
||||
result: Request | Response | None = None
|
||||
try:
|
||||
result = mw.process_request(request)
|
||||
if result:
|
||||
assert isinstance(result, (Request, Response))
|
||||
assert isinstance(result, Response)
|
||||
return result
|
||||
assert response is not None
|
||||
result = mw.process_response(request, response)
|
||||
|
|
@ -346,6 +351,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
|
|||
res2 = self._process_requestresponse(mw, req0, res0)
|
||||
assert "cached" not in res2.flags
|
||||
res3 = mw.process_request(req0)
|
||||
assert isinstance(res3, Response)
|
||||
assert "cached" in res3.flags
|
||||
self.assertEqualResponse(res2, res3)
|
||||
# request with no-cache directive must not return cached response
|
||||
|
|
@ -634,6 +640,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
|
|||
assert mw.process_request(req0) is None
|
||||
res1 = mw.process_exception(req0, e("foo"))
|
||||
# Use cached response as recovery
|
||||
assert isinstance(res1, Response)
|
||||
assert "cached" in res1.flags
|
||||
self.assertEqualResponse(res0, res1)
|
||||
# Do not use cached response for unhandled exceptions
|
||||
|
|
@ -684,26 +691,22 @@ class DbmStorageTestMixin(StorageTestMixin):
|
|||
|
||||
|
||||
class TestFilesystemStorageWithDummyPolicy(
|
||||
TestBase, FilesystemStorageTestMixin, DummyPolicyTestMixin
|
||||
FilesystemStorageTestMixin, DummyPolicyTestMixin
|
||||
):
|
||||
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
|
||||
|
||||
|
||||
class TestFilesystemStorageWithRFC2616Policy(
|
||||
TestBase, FilesystemStorageTestMixin, RFC2616PolicyTestMixin
|
||||
FilesystemStorageTestMixin, RFC2616PolicyTestMixin
|
||||
):
|
||||
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
|
||||
|
||||
|
||||
class TestDbmStorageWithDummyPolicy(
|
||||
TestBase, DbmStorageTestMixin, DummyPolicyTestMixin
|
||||
):
|
||||
class TestDbmStorageWithDummyPolicy(DbmStorageTestMixin, DummyPolicyTestMixin):
|
||||
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
|
||||
|
||||
|
||||
class TestDbmStorageWithRFC2616Policy(
|
||||
TestBase, DbmStorageTestMixin, RFC2616PolicyTestMixin
|
||||
):
|
||||
class TestDbmStorageWithRFC2616Policy(DbmStorageTestMixin, RFC2616PolicyTestMixin):
|
||||
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from importlib.util import find_spec
|
|||
from io import BytesIO
|
||||
from logging import WARNING
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from w3lib.encoding import resolve_encoding
|
||||
|
|
@ -15,6 +16,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWar
|
|||
from scrapy.http import HtmlResponse, Request, Response
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils._compression import _DecompressionMaxSizeExceeded
|
||||
from scrapy.utils.gz import gunzip
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests import tests_datadir
|
||||
|
|
@ -72,6 +74,7 @@ class TestHttpCompression:
|
|||
def setup_method(self):
|
||||
self.crawler = get_crawler(Spider)
|
||||
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
|
||||
assert self.crawler.stats
|
||||
self.crawler.stats.open_spider()
|
||||
|
||||
def _getresponse(self, coding: str) -> Response:
|
||||
|
|
@ -96,7 +99,8 @@ class TestHttpCompression:
|
|||
)
|
||||
return response
|
||||
|
||||
def assertStatsEqual(self, key, value):
|
||||
def assertStatsEqual(self, key: str, value: Any) -> None:
|
||||
assert self.crawler.stats
|
||||
assert self.crawler.stats.get_value(key) == value, str(
|
||||
self.crawler.stats.get_stats()
|
||||
)
|
||||
|
|
@ -145,6 +149,7 @@ class TestHttpCompression:
|
|||
|
||||
def test_process_response_gzip(self):
|
||||
response = self._getresponse("gzip")
|
||||
assert response.request
|
||||
request = response.request
|
||||
|
||||
assert response.headers["Content-Encoding"] == b"gzip"
|
||||
|
|
@ -159,6 +164,7 @@ class TestHttpCompression:
|
|||
_skip_if_no_br()
|
||||
|
||||
response = self._getresponse("br")
|
||||
assert response.request
|
||||
request = response.request
|
||||
assert response.headers["Content-Encoding"] == b"br"
|
||||
newresponse = self.mw.process_response(request, response)
|
||||
|
|
@ -172,6 +178,7 @@ class TestHttpCompression:
|
|||
if find_spec("brotli") is not None or find_spec("brotlicffi") is not None:
|
||||
pytest.skip("Requires not having brotli support")
|
||||
response = self._getresponse("br")
|
||||
assert response.request
|
||||
request = response.request
|
||||
assert response.headers["Content-Encoding"] == b"br"
|
||||
caplog.clear()
|
||||
|
|
@ -201,6 +208,7 @@ class TestHttpCompression:
|
|||
if not check_key.startswith("zstd-"):
|
||||
continue
|
||||
response = self._getresponse(check_key)
|
||||
assert response.request
|
||||
request = response.request
|
||||
assert response.headers["Content-Encoding"] == b"zstd"
|
||||
newresponse = self.mw.process_response(request, response)
|
||||
|
|
@ -216,6 +224,7 @@ class TestHttpCompression:
|
|||
if find_spec("zstandard") is not None:
|
||||
pytest.skip("Requires not having zstandard support")
|
||||
response = self._getresponse("zstd-static-content-size")
|
||||
assert response.request
|
||||
request = response.request
|
||||
assert response.headers["Content-Encoding"] == b"zstd"
|
||||
caplog.clear()
|
||||
|
|
@ -239,6 +248,7 @@ class TestHttpCompression:
|
|||
|
||||
def test_process_response_rawdeflate(self):
|
||||
response = self._getresponse("rawdeflate")
|
||||
assert response.request
|
||||
request = response.request
|
||||
|
||||
assert response.headers["Content-Encoding"] == b"deflate"
|
||||
|
|
@ -251,6 +261,7 @@ class TestHttpCompression:
|
|||
|
||||
def test_process_response_zlibdelate(self):
|
||||
response = self._getresponse("zlibdeflate")
|
||||
assert response.request
|
||||
request = response.request
|
||||
|
||||
assert response.headers["Content-Encoding"] == b"deflate"
|
||||
|
|
@ -275,6 +286,7 @@ class TestHttpCompression:
|
|||
def test_multipleencodings(self):
|
||||
response = self._getresponse("gzip")
|
||||
response.headers["Content-Encoding"] = ["uuencode", "gzip"]
|
||||
assert response.request
|
||||
request = response.request
|
||||
newresponse = self.mw.process_response(request, response)
|
||||
assert newresponse is not response
|
||||
|
|
@ -282,6 +294,7 @@ class TestHttpCompression:
|
|||
|
||||
def test_multi_compression_single_header(self):
|
||||
response = self._getresponse("gzip-deflate")
|
||||
assert response.request
|
||||
request = response.request
|
||||
newresponse = self.mw.process_response(request, response)
|
||||
assert newresponse is not response
|
||||
|
|
@ -293,6 +306,7 @@ class TestHttpCompression:
|
|||
) -> None:
|
||||
response = self._getresponse("gzip-deflate")
|
||||
response.headers["Content-Encoding"] = [b"gzip, foo, deflate"]
|
||||
assert response.request
|
||||
request = response.request
|
||||
caplog.clear()
|
||||
with caplog.at_level(
|
||||
|
|
@ -315,6 +329,7 @@ class TestHttpCompression:
|
|||
def test_multi_compression_multiple_header(self):
|
||||
response = self._getresponse("gzip-deflate")
|
||||
response.headers["Content-Encoding"] = ["gzip", "deflate"]
|
||||
assert response.request
|
||||
request = response.request
|
||||
newresponse = self.mw.process_response(request, response)
|
||||
assert newresponse is not response
|
||||
|
|
@ -324,6 +339,7 @@ class TestHttpCompression:
|
|||
def test_multi_compression_multiple_header_invalid_compression(self):
|
||||
response = self._getresponse("gzip-deflate")
|
||||
response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"]
|
||||
assert response.request
|
||||
request = response.request
|
||||
newresponse = self.mw.process_response(request, response)
|
||||
assert newresponse is not response
|
||||
|
|
@ -332,6 +348,7 @@ class TestHttpCompression:
|
|||
def test_multi_compression_single_and_multiple_header(self):
|
||||
response = self._getresponse("gzip-deflate-gzip")
|
||||
response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"]
|
||||
assert response.request
|
||||
request = response.request
|
||||
newresponse = self.mw.process_response(request, response)
|
||||
assert newresponse is not response
|
||||
|
|
@ -341,6 +358,7 @@ class TestHttpCompression:
|
|||
def test_multi_compression_single_and_multiple_header_invalid_compression(self):
|
||||
response = self._getresponse("gzip-deflate")
|
||||
response.headers["Content-Encoding"] = ["gzip", "foo,deflate"]
|
||||
assert response.request
|
||||
request = response.request
|
||||
newresponse = self.mw.process_response(request, response)
|
||||
assert newresponse is not response
|
||||
|
|
@ -397,9 +415,7 @@ class TestHttpCompression:
|
|||
self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
|
||||
|
||||
def test_process_response_no_content_type_header(self):
|
||||
headers = {
|
||||
"Content-Encoding": "identity",
|
||||
}
|
||||
headers = {b"Content-Encoding": b"identity"}
|
||||
plainbody = (
|
||||
b"<html><head><title>Some page</title>"
|
||||
b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">'
|
||||
|
|
@ -414,6 +430,7 @@ class TestHttpCompression:
|
|||
|
||||
newresponse = self.mw.process_response(request, response)
|
||||
assert isinstance(newresponse, respcls)
|
||||
assert isinstance(newresponse, HtmlResponse)
|
||||
assert newresponse.body == plainbody
|
||||
assert newresponse.encoding == resolve_encoding("gb2312")
|
||||
self.assertStatsEqual("httpcompression/response_count", 1)
|
||||
|
|
@ -422,6 +439,7 @@ class TestHttpCompression:
|
|||
def test_process_response_gzipped_contenttype(self):
|
||||
response = self._getresponse("gzip")
|
||||
response.headers["Content-Type"] = "application/gzip"
|
||||
assert response.request
|
||||
request = response.request
|
||||
|
||||
newresponse = self.mw.process_response(request, response)
|
||||
|
|
@ -434,6 +452,7 @@ class TestHttpCompression:
|
|||
def test_process_response_gzip_app_octetstream_contenttype(self):
|
||||
response = self._getresponse("gzip")
|
||||
response.headers["Content-Type"] = "application/octet-stream"
|
||||
assert response.request
|
||||
request = response.request
|
||||
|
||||
newresponse = self.mw.process_response(request, response)
|
||||
|
|
@ -446,6 +465,7 @@ class TestHttpCompression:
|
|||
def test_process_response_gzip_binary_octetstream_contenttype(self):
|
||||
response = self._getresponse("x-gzip")
|
||||
response.headers["Content-Type"] = "binary/octet-stream"
|
||||
assert response.request
|
||||
request = response.request
|
||||
|
||||
newresponse = self.mw.process_response(request, response)
|
||||
|
|
@ -504,6 +524,7 @@ class TestHttpCompression:
|
|||
def test_process_response_head_request_no_decode_required(self):
|
||||
response = self._getresponse("gzip")
|
||||
response.headers["Content-Type"] = "application/gzip"
|
||||
assert response.request
|
||||
request = response.request
|
||||
request.method = "HEAD"
|
||||
response = response.replace(body=None)
|
||||
|
|
@ -513,7 +534,7 @@ class TestHttpCompression:
|
|||
self.assertStatsEqual("httpcompression/response_count", None)
|
||||
self.assertStatsEqual("httpcompression/response_bytes", None)
|
||||
|
||||
def _test_compression_bomb_setting(self, compression_id):
|
||||
def _test_compression_bomb_setting(self, compression_id: str) -> None:
|
||||
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
|
||||
crawler = get_crawler(Spider, settings_dict=settings)
|
||||
spider = crawler._create_spider("scrapytest.org")
|
||||
|
|
@ -521,9 +542,12 @@ class TestHttpCompression:
|
|||
mw.open_spider(spider)
|
||||
|
||||
response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B
|
||||
assert response.request
|
||||
with pytest.raises(IgnoreRequest) as exc_info:
|
||||
mw.process_response(response.request, response)
|
||||
assert exc_info.value.__cause__.decompressed_size < 1_100_000
|
||||
cause = exc_info.value.__cause__
|
||||
assert isinstance(cause, _DecompressionMaxSizeExceeded)
|
||||
assert cause.decompressed_size < 1_100_000
|
||||
|
||||
def test_compression_bomb_setting_br(self):
|
||||
_skip_if_no_br()
|
||||
|
|
@ -549,6 +573,7 @@ class TestHttpCompression:
|
|||
mw.open_spider(spider)
|
||||
|
||||
response = self._getresponse("bomb-gzip") # 11_511_612 B
|
||||
assert response.request
|
||||
caplog.clear()
|
||||
with (
|
||||
caplog.at_level(
|
||||
|
|
@ -565,7 +590,7 @@ class TestHttpCompression:
|
|||
)
|
||||
]
|
||||
|
||||
def _test_compression_bomb_spider_attr(self, compression_id):
|
||||
def _test_compression_bomb_spider_attr(self, compression_id: str) -> None:
|
||||
class DownloadMaxSizeSpider(Spider):
|
||||
download_maxsize = 1_000_000
|
||||
|
||||
|
|
@ -575,9 +600,12 @@ class TestHttpCompression:
|
|||
mw.open_spider(spider)
|
||||
|
||||
response = self._getresponse(f"bomb-{compression_id}")
|
||||
assert response.request
|
||||
with pytest.raises(IgnoreRequest) as exc_info:
|
||||
mw.process_response(response.request, response)
|
||||
assert exc_info.value.__cause__.decompressed_size < 1_100_000
|
||||
cause = exc_info.value.__cause__
|
||||
assert isinstance(cause, _DecompressionMaxSizeExceeded)
|
||||
assert cause.decompressed_size < 1_100_000
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
def test_compression_bomb_spider_attr_br(self):
|
||||
|
|
@ -599,7 +627,7 @@ class TestHttpCompression:
|
|||
|
||||
self._test_compression_bomb_spider_attr("zstd")
|
||||
|
||||
def _test_compression_bomb_request_meta(self, compression_id):
|
||||
def _test_compression_bomb_request_meta(self, compression_id: str) -> None:
|
||||
crawler = get_crawler(Spider)
|
||||
spider = crawler._create_spider("scrapytest.org")
|
||||
mw = HttpCompressionMiddleware.from_crawler(crawler)
|
||||
|
|
@ -607,9 +635,12 @@ class TestHttpCompression:
|
|||
|
||||
response = self._getresponse(f"bomb-{compression_id}")
|
||||
response.meta["download_maxsize"] = 1_000_000
|
||||
assert response.request
|
||||
with pytest.raises(IgnoreRequest) as exc_info:
|
||||
mw.process_response(response.request, response)
|
||||
assert exc_info.value.__cause__.decompressed_size < 1_100_000
|
||||
cause = exc_info.value.__cause__
|
||||
assert isinstance(cause, _DecompressionMaxSizeExceeded)
|
||||
assert cause.decompressed_size < 1_100_000
|
||||
|
||||
def test_compression_bomb_request_meta_br(self):
|
||||
_skip_if_no_br()
|
||||
|
|
@ -789,7 +820,7 @@ class TestHttpCompression:
|
|||
|
||||
self._test_download_warnsize_request_meta(caplog, "zstd")
|
||||
|
||||
def _get_truncated_response(self, compression_id):
|
||||
def _get_truncated_response(self, compression_id: str) -> Response:
|
||||
crawler = get_crawler(Spider)
|
||||
spider = crawler._create_spider("scrapytest.org")
|
||||
mw = HttpCompressionMiddleware.from_crawler(crawler)
|
||||
|
|
@ -797,7 +828,10 @@ class TestHttpCompression:
|
|||
response = self._getresponse(compression_id)
|
||||
truncated_body = response.body[: len(response.body) // 2]
|
||||
response = response.replace(body=truncated_body)
|
||||
return mw.process_response(response.request, response)
|
||||
assert response.request
|
||||
new_response = mw.process_response(response.request, response)
|
||||
assert isinstance(new_response, Response)
|
||||
return new_response
|
||||
|
||||
def test_process_truncated_response_br(self):
|
||||
_skip_if_no_br()
|
||||
|
|
|
|||
|
|
@ -14,7 +14,8 @@ class TestHttpProxyMiddleware:
|
|||
self._oldenv = os.environ.copy()
|
||||
|
||||
def teardown_method(self):
|
||||
os.environ = self._oldenv
|
||||
os.environ.clear()
|
||||
os.environ.update(self._oldenv)
|
||||
|
||||
def test_not_enabled(self):
|
||||
crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False})
|
||||
|
|
@ -22,7 +23,8 @@ class TestHttpProxyMiddleware:
|
|||
HttpProxyMiddleware.from_crawler(crawler)
|
||||
|
||||
def test_no_environment_proxies(self):
|
||||
os.environ = {"dummy_proxy": "reset_env_and_do_not_raise"}
|
||||
os.environ.clear()
|
||||
os.environ["dummy_proxy"] = "reset_env_and_do_not_raise"
|
||||
mw = HttpProxyMiddleware()
|
||||
|
||||
for url in ("http://e.com", "https://e.com", "file:///tmp/a"):
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import re
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -53,7 +54,7 @@ def test_process_request_dont_filter(value, filtered):
|
|||
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
|
||||
mw = OffsiteMiddleware.from_crawler(crawler)
|
||||
mw.spider_opened(crawler.spider)
|
||||
kwargs = {}
|
||||
kwargs: dict[str, Any] = {}
|
||||
if value is not UNSET:
|
||||
kwargs["dont_filter"] = value
|
||||
request = Request("https://b.example", **kwargs)
|
||||
|
|
@ -82,7 +83,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
|
|||
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
|
||||
mw = OffsiteMiddleware.from_crawler(crawler)
|
||||
mw.spider_opened(crawler.spider)
|
||||
kwargs = {"meta": {}}
|
||||
kwargs: dict[str, Any] = {"meta": {}}
|
||||
if allow_offsite is not UNSET:
|
||||
kwargs["meta"]["allow_offsite"] = allow_offsite
|
||||
if dont_filter is not UNSET:
|
||||
|
|
@ -105,7 +106,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
|
|||
)
|
||||
def test_process_request_no_allowed_domains(value):
|
||||
crawler = get_crawler(Spider)
|
||||
kwargs = {}
|
||||
kwargs: dict[str, Any] = {}
|
||||
if value is not UNSET:
|
||||
kwargs["allowed_domains"] = value
|
||||
crawler.spider = crawler._create_spider(name="a", **kwargs)
|
||||
|
|
@ -152,7 +153,7 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
|
|||
mw.spider_opened(crawler.spider)
|
||||
request = Request(url)
|
||||
if allowed:
|
||||
assert mw.request_scheduled(request, crawler.spider) is None
|
||||
mw.request_scheduled(request, crawler.spider)
|
||||
else:
|
||||
with pytest.raises(IgnoreRequest):
|
||||
mw.request_scheduled(request, crawler.spider)
|
||||
|
|
@ -172,7 +173,7 @@ def test_request_scheduled_dont_filter(value, filtered):
|
|||
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
|
||||
mw = OffsiteMiddleware.from_crawler(crawler)
|
||||
mw.spider_opened(crawler.spider)
|
||||
kwargs = {}
|
||||
kwargs: dict[str, Any] = {}
|
||||
if value is not UNSET:
|
||||
kwargs["dont_filter"] = value
|
||||
request = Request("https://b.example", **kwargs)
|
||||
|
|
@ -180,7 +181,7 @@ def test_request_scheduled_dont_filter(value, filtered):
|
|||
with pytest.raises(IgnoreRequest):
|
||||
mw.request_scheduled(request, crawler.spider)
|
||||
else:
|
||||
assert mw.request_scheduled(request, crawler.spider) is None
|
||||
mw.request_scheduled(request, crawler.spider)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
@ -193,14 +194,14 @@ def test_request_scheduled_dont_filter(value, filtered):
|
|||
)
|
||||
def test_request_scheduled_no_allowed_domains(value):
|
||||
crawler = get_crawler(Spider)
|
||||
kwargs = {}
|
||||
kwargs: dict[str, Any] = {}
|
||||
if value is not UNSET:
|
||||
kwargs["allowed_domains"] = value
|
||||
crawler.spider = crawler._create_spider(name="a", **kwargs)
|
||||
mw = OffsiteMiddleware.from_crawler(crawler)
|
||||
mw.spider_opened(crawler.spider)
|
||||
request = Request("https://example.com")
|
||||
assert mw.request_scheduled(request, crawler.spider) is None
|
||||
mw.request_scheduled(request, crawler.spider)
|
||||
|
||||
|
||||
def test_request_scheduled_invalid_domains():
|
||||
|
|
@ -210,7 +211,7 @@ def test_request_scheduled_invalid_domains():
|
|||
mw = OffsiteMiddleware.from_crawler(crawler)
|
||||
mw.spider_opened(crawler.spider)
|
||||
request = Request("https://a.example")
|
||||
assert mw.request_scheduled(request, crawler.spider) is None
|
||||
mw.request_scheduled(request, crawler.spider)
|
||||
for letter in ("b", "c"):
|
||||
request = Request(f"https://{letter}.example")
|
||||
with pytest.raises(IgnoreRequest):
|
||||
|
|
@ -227,6 +228,7 @@ def test_repeated_offsite_domain():
|
|||
with pytest.raises(IgnoreRequest):
|
||||
mw.process_request(req1)
|
||||
assert "other.org" in mw.domains_seen
|
||||
assert crawler.stats
|
||||
assert crawler.stats.get_value("offsite/domains") == 1
|
||||
assert crawler.stats.get_value("offsite/filtered") == 1
|
||||
with pytest.raises(IgnoreRequest):
|
||||
|
|
|
|||
|
|
@ -309,7 +309,7 @@ class TestRedirectMiddleware(TestRedirectBase):
|
|||
url = "http://www.example.com/301"
|
||||
url2 = "http://www.example.com/redirected"
|
||||
|
||||
def _test_passthrough(req):
|
||||
def _test_passthrough(req: Request) -> None:
|
||||
rsp = Response(url, headers={"Location": url2}, status=301, request=req)
|
||||
r = self.mw.process_response(req, rsp)
|
||||
assert r is rsp
|
||||
|
|
@ -404,15 +404,17 @@ def test_response_referrer_policy(policy, source_url, target_url, expected_refer
|
|||
status=301,
|
||||
headers={"Location": target_url, **extra_headers},
|
||||
)
|
||||
source_request = redirect_mw.process_response(source_request, response_redirect)
|
||||
assert isinstance(source_request, Request)
|
||||
target_request = redirect_mw.process_response(source_request, response_redirect)
|
||||
assert isinstance(target_request, Request)
|
||||
|
||||
assert source_request.headers.get("Referer") == expected_referrer
|
||||
assert target_request.headers.get("Referer") == expected_referrer
|
||||
|
||||
|
||||
def test_no_warning_when_referer_middleware_present(caplog):
|
||||
crawler = get_crawler()
|
||||
crawler.get_spider_middleware = MagicMock(return_value=MagicMock())
|
||||
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
|
||||
return_value=MagicMock()
|
||||
)
|
||||
mw = build_from_crawler(RedirectMiddleware, crawler)
|
||||
caplog.clear()
|
||||
with caplog.at_level(logging.WARNING):
|
||||
|
|
@ -426,7 +428,9 @@ def test_no_warning_when_referer_middleware_present(caplog):
|
|||
|
||||
def test_warning_redirect_middleware(caplog):
|
||||
crawler = get_crawler()
|
||||
crawler.get_spider_middleware = MagicMock(return_value=None)
|
||||
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
|
||||
return_value=None
|
||||
)
|
||||
mw = build_from_crawler(RedirectMiddleware, crawler)
|
||||
with caplog.at_level(logging.WARNING):
|
||||
mw._engine_started()
|
||||
|
|
@ -449,7 +453,9 @@ def test_warning_subclass(caplog):
|
|||
pass
|
||||
|
||||
crawler = get_crawler()
|
||||
crawler.get_spider_middleware = MagicMock(return_value=None)
|
||||
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
|
||||
return_value=None
|
||||
)
|
||||
mw = build_from_crawler(MyRedirectMiddleware, crawler)
|
||||
with caplog.at_level(logging.WARNING):
|
||||
mw._engine_started()
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ from tests.utils.redirect import (
|
|||
)
|
||||
|
||||
|
||||
def meta_refresh_body(url, interval=5):
|
||||
def meta_refresh_body(url: str, interval: int = 5) -> bytes:
|
||||
html = f"""<html><head><meta http-equiv="refresh" content="{interval};url={url}"/></head></html>"""
|
||||
return html.encode("utf-8")
|
||||
|
||||
|
|
@ -34,10 +34,14 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
|
|||
crawler = get_crawler(Spider)
|
||||
self.mw = self.mwcls.from_crawler(crawler)
|
||||
|
||||
def _body(self, interval=5, url="http://example.org/newpage"):
|
||||
def _body(
|
||||
self, interval: int = 5, url: str = "http://example.org/newpage"
|
||||
) -> bytes:
|
||||
return meta_refresh_body(url, interval)
|
||||
|
||||
def get_response(self, request, location):
|
||||
def get_response(
|
||||
self, request: Request, location: str, status: int = 302
|
||||
) -> Response:
|
||||
return HtmlResponse(request.url, body=self._body(url=location))
|
||||
|
||||
def test_meta_refresh(self):
|
||||
|
|
@ -75,7 +79,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
|
|||
assert "Content-Length" not in req2.headers, (
|
||||
"Content-Length header must not be present in redirected request"
|
||||
)
|
||||
assert not req2.body, f"Redirected body must be empty, not '{req2.body}'"
|
||||
assert not req2.body, f"Redirected body must be empty, not {req2.body!r}"
|
||||
|
||||
def test_ignore_tags_default(self):
|
||||
req = Request(url="http://example.org")
|
||||
|
|
@ -142,7 +146,9 @@ def test_meta_refresh_schemes(url, location, target):
|
|||
|
||||
def test_warning_meta_refresh_middleware(caplog):
|
||||
crawler = get_crawler()
|
||||
crawler.get_spider_middleware = MagicMock(return_value=None)
|
||||
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
|
||||
return_value=None
|
||||
)
|
||||
mw = build_from_crawler(MetaRefreshMiddleware, crawler)
|
||||
with caplog.at_level(logging.WARNING):
|
||||
mw._engine_started()
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ class TestRetry:
|
|||
req = Request("http://www.scrapytest.org/503")
|
||||
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
|
||||
req2 = self.mw.process_response(req, rsp)
|
||||
assert isinstance(req2, Request)
|
||||
assert req2.priority < req.priority
|
||||
|
||||
def test_404(self):
|
||||
|
|
@ -53,9 +54,9 @@ class TestRetry:
|
|||
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
|
||||
|
||||
# first retry
|
||||
req = self.mw.process_response(req, rsp)
|
||||
assert isinstance(req, Request)
|
||||
assert req.meta["retry_times"] == 1
|
||||
req2 = self.mw.process_response(req, rsp)
|
||||
assert isinstance(req2, Request)
|
||||
assert req2.meta["retry_times"] == 1
|
||||
|
||||
def test_dont_retry_exc(self):
|
||||
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
|
||||
|
|
@ -68,18 +69,19 @@ class TestRetry:
|
|||
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
|
||||
|
||||
# first retry
|
||||
req = self.mw.process_response(req, rsp)
|
||||
assert isinstance(req, Request)
|
||||
assert req.meta["retry_times"] == 1
|
||||
req2 = self.mw.process_response(req, rsp)
|
||||
assert isinstance(req2, Request)
|
||||
assert req2.meta["retry_times"] == 1
|
||||
|
||||
# second retry
|
||||
req = self.mw.process_response(req, rsp)
|
||||
assert isinstance(req, Request)
|
||||
assert req.meta["retry_times"] == 2
|
||||
req3 = self.mw.process_response(req2, rsp)
|
||||
assert isinstance(req3, Request)
|
||||
assert req3.meta["retry_times"] == 2
|
||||
|
||||
# discard it
|
||||
assert self.mw.process_response(req, rsp) is rsp
|
||||
assert self.mw.process_response(req3, rsp) is rsp
|
||||
|
||||
assert self.crawler.stats
|
||||
assert self.crawler.stats.get_value("retry/max_reached") == 1
|
||||
assert (
|
||||
self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable")
|
||||
|
|
@ -131,6 +133,7 @@ class TestRetry:
|
|||
self._test_retry_exception(req, exc("foo"))
|
||||
|
||||
stats = self.crawler.stats
|
||||
assert stats
|
||||
assert stats.get_value("retry/max_reached") == len(exceptions)
|
||||
assert stats.get_value("retry/count") == len(exceptions) * 2
|
||||
assert (
|
||||
|
|
@ -149,29 +152,30 @@ class TestRetry:
|
|||
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
|
||||
self._test_retry_exception(req, exc("foo"), mw)
|
||||
|
||||
def _test_retry_exception(self, req, exception, mw=None):
|
||||
def _test_retry_exception(
|
||||
self, req: Request, exception: Exception, mw: RetryMiddleware | None = None
|
||||
) -> None:
|
||||
if mw is None:
|
||||
mw = self.mw
|
||||
|
||||
# first retry
|
||||
req = mw.process_exception(req, exception)
|
||||
assert isinstance(req, Request)
|
||||
assert req.meta["retry_times"] == 1
|
||||
req2 = mw.process_exception(req, exception)
|
||||
assert isinstance(req2, Request)
|
||||
assert req2.meta["retry_times"] == 1
|
||||
|
||||
# second retry
|
||||
req = mw.process_exception(req, exception)
|
||||
assert isinstance(req, Request)
|
||||
assert req.meta["retry_times"] == 2
|
||||
req3 = mw.process_exception(req2, exception)
|
||||
assert isinstance(req3, Request)
|
||||
assert req3.meta["retry_times"] == 2
|
||||
|
||||
# discard it
|
||||
req = mw.process_exception(req, exception)
|
||||
assert req is None
|
||||
assert mw.process_exception(req3, exception) is None
|
||||
|
||||
|
||||
class TestMaxRetryTimes:
|
||||
invalid_url = "http://www.scrapytest.org/invalid_url"
|
||||
|
||||
def get_middleware(self, settings=None):
|
||||
def get_middleware(self, settings: dict[str, Any] | None = None) -> RetryMiddleware:
|
||||
crawler = get_crawler(DefaultSpider, settings or {})
|
||||
crawler.spider = crawler._create_spider()
|
||||
return RetryMiddleware.from_crawler(crawler)
|
||||
|
|
@ -275,20 +279,18 @@ class TestMaxRetryTimes:
|
|||
|
||||
def _test_retry(
|
||||
self,
|
||||
req,
|
||||
exception,
|
||||
max_retry_times,
|
||||
middleware=None,
|
||||
):
|
||||
middleware = middleware or self.mw
|
||||
|
||||
req: Request,
|
||||
exception: Exception,
|
||||
max_retry_times: int,
|
||||
middleware: RetryMiddleware,
|
||||
) -> None:
|
||||
for _ in range(max_retry_times):
|
||||
req = middleware.process_exception(req, exception)
|
||||
assert isinstance(req, Request)
|
||||
result = middleware.process_exception(req, exception)
|
||||
assert isinstance(result, Request)
|
||||
req = result
|
||||
|
||||
# discard it
|
||||
req = middleware.process_exception(req, exception)
|
||||
assert req is None
|
||||
assert middleware.process_exception(req, exception) is None
|
||||
|
||||
|
||||
class TestGetRetryRequest:
|
||||
|
|
@ -428,7 +430,7 @@ class TestGetRetryRequest:
|
|||
def test_no_spider(self):
|
||||
request = Request("https://example.com")
|
||||
with pytest.raises(TypeError):
|
||||
get_retry_request(request) # pylint: disable=missing-kwoa
|
||||
get_retry_request(request) # type: ignore[call-arg] # pylint: disable=missing-kwoa
|
||||
|
||||
def test_max_retry_times_setting(self):
|
||||
max_retry_times = 0
|
||||
|
|
@ -471,6 +473,7 @@ class TestGetRetryRequest:
|
|||
request,
|
||||
spider=spider,
|
||||
)
|
||||
assert new_request
|
||||
assert new_request.priority == priority_adjust
|
||||
|
||||
def test_priority_adjust_argument(self):
|
||||
|
|
@ -482,6 +485,7 @@ class TestGetRetryRequest:
|
|||
spider=spider,
|
||||
priority_adjust=priority_adjust,
|
||||
)
|
||||
assert new_request
|
||||
assert new_request.priority == priority_adjust
|
||||
|
||||
def test_log_extra_retry_success(self, caplog: pytest.LogCaptureFixture) -> None:
|
||||
|
|
@ -732,6 +736,7 @@ class TestGetRetryRequest:
|
|||
reason=expected_reason,
|
||||
stats_base_key=stats_key,
|
||||
)
|
||||
assert spider.crawler.stats
|
||||
for stat in (
|
||||
f"{stats_key}/count",
|
||||
f"{stats_key}/reason_count/{expected_reason}",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import TYPE_CHECKING
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -19,9 +18,6 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
|
|||
from tests.utils.decorators import coroutine_test
|
||||
from tests.utils.robotstxt import rerp_available
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
class TestRobotsTxtMiddleware:
|
||||
def setup_method(self) -> None:
|
||||
|
|
@ -39,7 +35,7 @@ class TestRobotsTxtMiddleware:
|
|||
with pytest.raises(NotConfigured):
|
||||
RobotsTxtMiddleware(self.crawler)
|
||||
|
||||
def _get_successful_crawler(self) -> Crawler:
|
||||
def _get_successful_crawler(self) -> mock.MagicMock:
|
||||
crawler = self.crawler
|
||||
crawler.settings.set("ROBOTSTXT_OBEY", True)
|
||||
ROBOTS = """
|
||||
|
|
@ -54,8 +50,8 @@ Disallow: /some/randome/page.html
|
|||
""".encode()
|
||||
response = TextResponse("http://site.local/robots.txt", body=ROBOTS)
|
||||
|
||||
async def return_response(request):
|
||||
deferred = Deferred()
|
||||
async def return_response(request: Request) -> Response:
|
||||
deferred: Deferred[Response] = Deferred()
|
||||
call_later(0, deferred.callback, response)
|
||||
return await maybe_deferred_to_future(deferred)
|
||||
|
||||
|
|
@ -130,15 +126,15 @@ Disallow: /some/randome/page.html
|
|||
Request("http://site.local/static/", meta=meta), middleware
|
||||
)
|
||||
|
||||
def _get_garbage_crawler(self) -> Crawler:
|
||||
def _get_garbage_crawler(self) -> mock.MagicMock:
|
||||
crawler = self.crawler
|
||||
crawler.settings.set("ROBOTSTXT_OBEY", True)
|
||||
response = Response(
|
||||
"http://site.local/robots.txt", body=b"GIF89a\xd3\x00\xfe\x00\xa2"
|
||||
)
|
||||
|
||||
async def return_response(request):
|
||||
deferred = Deferred()
|
||||
async def return_response(request: Request) -> Response:
|
||||
deferred: Deferred[Response] = Deferred()
|
||||
call_later(0, deferred.callback, response)
|
||||
return await maybe_deferred_to_future(deferred)
|
||||
|
||||
|
|
@ -154,13 +150,13 @@ Disallow: /some/randome/page.html
|
|||
await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)
|
||||
await self.assertNotIgnored(Request("http://site.local/static/"), middleware)
|
||||
|
||||
def _get_emptybody_crawler(self) -> Crawler:
|
||||
def _get_emptybody_crawler(self) -> mock.MagicMock:
|
||||
crawler = self.crawler
|
||||
crawler.settings.set("ROBOTSTXT_OBEY", True)
|
||||
response = Response("http://site.local/robots.txt")
|
||||
|
||||
async def return_response(request):
|
||||
deferred = Deferred()
|
||||
async def return_response(request: Request) -> Response:
|
||||
deferred: Deferred[Response] = Deferred()
|
||||
call_later(0, deferred.callback, response)
|
||||
return await maybe_deferred_to_future(deferred)
|
||||
|
||||
|
|
@ -180,8 +176,8 @@ Disallow: /some/randome/page.html
|
|||
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
|
||||
err = CannotResolveHostError("Robotstxt address not found")
|
||||
|
||||
async def return_failure(request):
|
||||
deferred = Deferred()
|
||||
async def return_failure(request: Request) -> Response:
|
||||
deferred: Deferred[Response] = Deferred()
|
||||
call_later(0, deferred.errback, failure.Failure(err))
|
||||
return await maybe_deferred_to_future(deferred)
|
||||
|
||||
|
|
@ -208,8 +204,8 @@ Disallow: /some/randome/page.html
|
|||
async def test_ignore_robotstxt_request(self):
|
||||
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
|
||||
|
||||
async def ignore_request(request):
|
||||
deferred = Deferred()
|
||||
async def ignore_request(request: Request) -> Response:
|
||||
deferred: Deferred[Response] = Deferred()
|
||||
call_later(0, deferred.errback, failure.Failure(IgnoreRequest()))
|
||||
return await maybe_deferred_to_future(deferred)
|
||||
|
||||
|
|
@ -236,7 +232,7 @@ Disallow: /some/randome/page.html
|
|||
@coroutine_test
|
||||
async def test_robotstxt_local_file(self):
|
||||
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
|
||||
middleware.process_request_2 = mock.MagicMock()
|
||||
middleware.process_request_2 = mock.MagicMock() # type: ignore[method-assign]
|
||||
|
||||
await middleware.process_request(Request("data:text/plain,Hello World data"))
|
||||
assert not middleware.process_request_2.called
|
||||
|
|
|
|||
|
|
@ -116,6 +116,18 @@ class TestEngineDownloadAsync:
|
|||
engine._slot.add_request.assert_called_once_with(request)
|
||||
engine._slot.remove_request.assert_called_once_with(request)
|
||||
|
||||
@coroutine_test
|
||||
async def test_download_async_fetch_needs_spider(self, engine):
|
||||
engine._downloader_fetch_needs_spider = True
|
||||
request = Request("http://example.com")
|
||||
response = Response("http://example.com", body=b"test body")
|
||||
engine.spider = Mock()
|
||||
engine.downloader.fetch.return_value = defer.succeed(response)
|
||||
|
||||
result = await self._download(engine, request)
|
||||
assert result == response
|
||||
engine.downloader.fetch.assert_called_once_with(request, engine.spider)
|
||||
|
||||
|
||||
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
|
||||
class TestEngineDownload(TestEngineDownloadAsync):
|
||||
|
|
|
|||
|
|
@ -6,10 +6,9 @@ from typing import TYPE_CHECKING, Any
|
|||
|
||||
from scrapy import Request, Spider, signals
|
||||
from scrapy.core.scheduler import BaseScheduler
|
||||
from scrapy.utils.asyncio import call_later
|
||||
from scrapy.utils.asyncio import call_later, sleep
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.mockserver.http import MockServer
|
||||
from tests.utils import async_sleep
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -65,23 +64,23 @@ class TestMain:
|
|||
async def start(self):
|
||||
yield Request("data:,a")
|
||||
|
||||
await async_sleep(seconds)
|
||||
await sleep(seconds)
|
||||
|
||||
self.crawler.engine._slot.scheduler.pause()
|
||||
self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))
|
||||
|
||||
# During this time, the scheduler reports having requests but
|
||||
# returns None.
|
||||
await async_sleep(seconds)
|
||||
await sleep(seconds)
|
||||
|
||||
self.crawler.engine._slot.scheduler.unpause()
|
||||
|
||||
# The scheduler request is processed.
|
||||
await async_sleep(seconds)
|
||||
await sleep(seconds)
|
||||
|
||||
yield Request("data:,c")
|
||||
|
||||
await async_sleep(seconds)
|
||||
await sleep(seconds)
|
||||
|
||||
self.crawler.engine._slot.scheduler.pause()
|
||||
self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d"))
|
||||
|
|
|
|||
|
|
@ -6,9 +6,6 @@ from scrapy.http import Headers
|
|||
|
||||
|
||||
class TestHeaders:
|
||||
def assertSortedEqual(self, first, second, msg=None):
|
||||
assert sorted(first) == sorted(second), msg
|
||||
|
||||
def test_basics(self):
|
||||
h = Headers({"Content-Type": "text/html", "Content-Length": 1234})
|
||||
assert h["Content-Type"]
|
||||
|
|
@ -39,7 +36,7 @@ class TestHeaders:
|
|||
assert h["X-Forwarded-For"] == b"ip2"
|
||||
assert h.get("X-Forwarded-For") == b"ip2"
|
||||
assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]
|
||||
assert h.getlist("X-Forwarded-For") is not hlist
|
||||
assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]
|
||||
|
||||
def test_multivalue_for_one_header(self):
|
||||
h = Headers((("a", "b"), ("a", "c")))
|
||||
|
|
@ -49,19 +46,19 @@ class TestHeaders:
|
|||
|
||||
def test_encode_utf8(self):
|
||||
h = Headers({"key": "\xa3"}, encoding="utf-8")
|
||||
key, val = dict(h).popitem()
|
||||
key, val = dict(h.items()).popitem()
|
||||
assert isinstance(key, bytes), key
|
||||
assert isinstance(val[0], bytes), val[0]
|
||||
assert val[0] == b"\xc2\xa3"
|
||||
|
||||
def test_encode_latin1(self):
|
||||
h = Headers({"key": "\xa3"}, encoding="latin1")
|
||||
_, val = dict(h).popitem()
|
||||
_, val = dict(h.items()).popitem()
|
||||
assert val[0] == b"\xa3"
|
||||
|
||||
def test_encode_multiple(self):
|
||||
h = Headers({"key": ["\xa3"]}, encoding="utf-8")
|
||||
_, val = dict(h).popitem()
|
||||
_, val = dict(h.items()).popitem()
|
||||
assert val[0] == b"\xc2\xa3"
|
||||
|
||||
def test_delete_and_contains(self):
|
||||
|
|
@ -75,7 +72,7 @@ class TestHeaders:
|
|||
h = Headers()
|
||||
hlist = ["ip1", "ip2"]
|
||||
olist = h.setdefault("X-Forwarded-For", hlist)
|
||||
assert h.getlist("X-Forwarded-For") is not hlist
|
||||
assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]
|
||||
assert h.getlist("X-Forwarded-For") is olist
|
||||
|
||||
h = Headers()
|
||||
|
|
@ -87,16 +84,16 @@ class TestHeaders:
|
|||
idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}
|
||||
|
||||
h = Headers(idict)
|
||||
assert dict(h) == {
|
||||
assert dict(h.items()) == {
|
||||
b"Content-Type": [b"text/html"],
|
||||
b"X-Forwarded-For": [b"ip1", b"ip2"],
|
||||
}
|
||||
self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"])
|
||||
self.assertSortedEqual(
|
||||
h.items(),
|
||||
[(b"X-Forwarded-For", [b"ip1", b"ip2"]), (b"Content-Type", [b"text/html"])],
|
||||
)
|
||||
self.assertSortedEqual(h.values(), [b"ip2", b"text/html"])
|
||||
assert sorted(h.keys()) == [b"Content-Type", b"X-Forwarded-For"]
|
||||
assert sorted(h.items()) == [
|
||||
(b"Content-Type", [b"text/html"]),
|
||||
(b"X-Forwarded-For", [b"ip1", b"ip2"]),
|
||||
]
|
||||
assert set(h.values()) == {b"ip2", b"text/html"}
|
||||
|
||||
def test_update(self):
|
||||
h = Headers()
|
||||
|
|
@ -162,4 +159,4 @@ class TestHeaders:
|
|||
with pytest.raises(TypeError, match="Unsupported value type"):
|
||||
Headers().setdefault("foo", object())
|
||||
with pytest.raises(TypeError, match="Unsupported value type"):
|
||||
Headers().setlist("foo", [object()])
|
||||
Headers().setlist("foo", [object()]) # type: ignore[list-item]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import xmlrpc.client
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -17,7 +18,7 @@ class TestXmlRpcRequest(TestRequestBase):
|
|||
default_method = "POST"
|
||||
default_headers = {b"Content-Type": [b"text/xml"]}
|
||||
|
||||
def _test_request(self, **kwargs):
|
||||
def _test_request(self, **kwargs: Any) -> None:
|
||||
r = self.request_class("http://scrapytest.org/rpc2", **kwargs)
|
||||
assert r.headers[b"Content-Type"] == b"text/xml"
|
||||
assert r.body == to_bytes(
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ from __future__ import annotations
|
|||
|
||||
import re
|
||||
import warnings
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from urllib.parse import parse_qs, unquote_to_bytes
|
||||
|
||||
import pytest
|
||||
|
|
@ -12,20 +13,32 @@ from scrapy.utils.httpobj import urlparse_cached
|
|||
from scrapy.utils.python import to_unicode
|
||||
from tests.utils.bases.http_request import TestRequestBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy import Request
|
||||
|
||||
def _buildresponse(body, **kwargs):
|
||||
|
||||
def _buildresponse(body: bytes | str, **kwargs: Any) -> HtmlResponse:
|
||||
kwargs.setdefault("body", body)
|
||||
kwargs.setdefault("url", "http://example.com")
|
||||
kwargs.setdefault("encoding", "utf-8")
|
||||
return HtmlResponse(**kwargs)
|
||||
|
||||
|
||||
def _qs(req, encoding="utf-8", to_unicode=False):
|
||||
qs = req.body if req.method == "POST" else req.url.partition("?")[2]
|
||||
uqs = unquote_to_bytes(qs)
|
||||
if to_unicode:
|
||||
uqs = uqs.decode(encoding)
|
||||
return parse_qs(uqs, True)
|
||||
def _query_string(req: Request) -> bytes:
|
||||
return req.body if req.method == "POST" else req.url.partition("?")[2].encode()
|
||||
|
||||
|
||||
def _qs(req: Request) -> dict[bytes, list[bytes]]:
|
||||
return parse_qs(unquote_to_bytes(_query_string(req)), True)
|
||||
|
||||
|
||||
def _qs_unicode(req: Request, encoding: str = "utf-8") -> dict[str, list[str]]:
|
||||
qs = unquote_to_bytes(_query_string(req)).decode(encoding)
|
||||
return parse_qs(qs, True)
|
||||
|
||||
|
||||
def _assert_query_equal(first: bytes, second: bytes) -> None:
|
||||
assert sorted(to_unicode(first).split("&")) == sorted(to_unicode(second).split("&"))
|
||||
|
||||
|
||||
# FormRequest.from_response() is deprecated in favor of form2request, so the
|
||||
|
|
@ -34,11 +47,6 @@ def _qs(req, encoding="utf-8", to_unicode=False):
|
|||
class TestFormRequest(TestRequestBase):
|
||||
request_class = FormRequest
|
||||
|
||||
def assertQueryEqual(self, first, second, msg=None):
|
||||
first = to_unicode(first).split("&")
|
||||
second = to_unicode(second).split("&")
|
||||
assert sorted(first) == sorted(second), msg
|
||||
|
||||
def test_init_not_deprecated(self):
|
||||
# Building a request directly from form data is not deprecated.
|
||||
with warnings.catch_warnings():
|
||||
|
|
@ -75,20 +83,22 @@ class TestFormRequest(TestRequestBase):
|
|||
assert fs[b"b"] == [b"2"]
|
||||
assert fs.get(b"c") is None
|
||||
|
||||
data = {"a": "1", "b": "2"}
|
||||
mapping = {"a": "1", "b": "2"}
|
||||
fs = _qs(
|
||||
self.request_class("http://www.example.com/", method="GET", formdata=data)
|
||||
self.request_class(
|
||||
"http://www.example.com/", method="GET", formdata=mapping
|
||||
)
|
||||
)
|
||||
assert fs[b"a"] == [b"1"]
|
||||
assert fs[b"b"] == [b"2"]
|
||||
|
||||
def test_default_encoding_bytes(self):
|
||||
# using default encoding (utf-8)
|
||||
data = {b"one": b"two", b"price": b"\xc2\xa3 100"}
|
||||
data: dict[Any, Any] = {b"one": b"two", b"price": b"\xc2\xa3 100"}
|
||||
r2 = self.request_class("http://www.example.com", formdata=data)
|
||||
assert r2.method == "POST"
|
||||
assert r2.encoding == "utf-8"
|
||||
self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two")
|
||||
_assert_query_equal(r2.body, b"price=%C2%A3+100&one=two")
|
||||
assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded"
|
||||
|
||||
def test_default_encoding_textual_data(self):
|
||||
|
|
@ -97,26 +107,26 @@ class TestFormRequest(TestRequestBase):
|
|||
r2 = self.request_class("http://www.example.com", formdata=data)
|
||||
assert r2.method == "POST"
|
||||
assert r2.encoding == "utf-8"
|
||||
self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two")
|
||||
_assert_query_equal(r2.body, b"price=%C2%A3+100&%C2%B5+one=two")
|
||||
assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded"
|
||||
|
||||
def test_default_encoding_mixed_data(self):
|
||||
# using default encoding (utf-8)
|
||||
data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"}
|
||||
data: dict[Any, Any] = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"}
|
||||
r2 = self.request_class("http://www.example.com", formdata=data)
|
||||
assert r2.method == "POST"
|
||||
assert r2.encoding == "utf-8"
|
||||
self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100")
|
||||
_assert_query_equal(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100")
|
||||
assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded"
|
||||
|
||||
def test_custom_encoding_bytes(self):
|
||||
data = {b"\xb5 one": b"two", b"price": b"\xa3 100"}
|
||||
data: dict[Any, Any] = {b"\xb5 one": b"two", b"price": b"\xa3 100"}
|
||||
r2 = self.request_class(
|
||||
"http://www.example.com", formdata=data, encoding="latin1"
|
||||
)
|
||||
assert r2.method == "POST"
|
||||
assert r2.encoding == "latin1"
|
||||
self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two")
|
||||
_assert_query_equal(r2.body, b"price=%A3+100&%B5+one=two")
|
||||
assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded"
|
||||
|
||||
def test_custom_encoding_textual_data(self):
|
||||
|
|
@ -131,7 +141,7 @@ class TestFormRequest(TestRequestBase):
|
|||
# using multiples values for a single key
|
||||
data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]}
|
||||
r3 = self.request_class("http://www.example.com", formdata=data)
|
||||
self.assertQueryEqual(
|
||||
_assert_query_equal(
|
||||
r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100"
|
||||
)
|
||||
|
||||
|
|
@ -173,7 +183,7 @@ class TestFormRequest(TestRequestBase):
|
|||
assert req.method == "POST"
|
||||
assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded"
|
||||
assert req.url == "http://www.example.com/this/post.php"
|
||||
fs = _qs(req, to_unicode=True)
|
||||
fs = _qs_unicode(req)
|
||||
assert set(fs["test £"]) == {"val1", "val2"}
|
||||
assert set(fs["one"]) == {"two", "three"}
|
||||
assert fs["test2"] == ["xxx µ"]
|
||||
|
|
@ -196,7 +206,7 @@ class TestFormRequest(TestRequestBase):
|
|||
assert req.method == "POST"
|
||||
assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded"
|
||||
assert req.url == "http://www.example.com/this/post.php"
|
||||
fs = _qs(req, to_unicode=True, encoding="latin1")
|
||||
fs = _qs_unicode(req, encoding="latin1")
|
||||
assert set(fs["test £"]) == {"val1", "val2"}
|
||||
assert set(fs["one"]) == {"two", "three"}
|
||||
assert fs["test2"] == ["xxx µ"]
|
||||
|
|
@ -218,7 +228,7 @@ class TestFormRequest(TestRequestBase):
|
|||
assert req.method == "POST"
|
||||
assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded"
|
||||
assert req.url == "http://www.example.com/this/post.php"
|
||||
fs = _qs(req, to_unicode=True)
|
||||
fs = _qs_unicode(req)
|
||||
assert set(fs["test £"]) == {"val1", "val2"}
|
||||
assert set(fs["one"]) == {"two", "three"}
|
||||
assert fs["test2"] == ["xxx µ"]
|
||||
|
|
@ -305,7 +315,10 @@ class TestFormRequest(TestRequestBase):
|
|||
<input type="hidden" name="two" value="3">
|
||||
</form>"""
|
||||
)
|
||||
req = self.request_class.from_response(response, formdata={"two": None})
|
||||
req = self.request_class.from_response(
|
||||
response,
|
||||
formdata={"two": None}, # type: ignore[arg-type]
|
||||
)
|
||||
fs = _qs(req)
|
||||
assert fs[b"one"] == [b"1"]
|
||||
assert b"two" not in fs
|
||||
|
|
@ -450,7 +463,7 @@ class TestFormRequest(TestRequestBase):
|
|||
req = self.request_class.from_response(
|
||||
response, clickdata={"name": "price in \u00a3"}
|
||||
)
|
||||
fs = _qs(req, to_unicode=True)
|
||||
fs = _qs_unicode(req)
|
||||
assert fs["price in \u00a3"]
|
||||
|
||||
def test_from_response_unicode_clickdata_latin1(self):
|
||||
|
|
@ -466,7 +479,7 @@ class TestFormRequest(TestRequestBase):
|
|||
req = self.request_class.from_response(
|
||||
response, clickdata={"name": "price in \u00a5"}
|
||||
)
|
||||
fs = _qs(req, to_unicode=True, encoding="latin1")
|
||||
fs = _qs_unicode(req, encoding="latin1")
|
||||
assert fs["price in \u00a5"]
|
||||
|
||||
def test_from_response_multiple_forms_clickdata(self):
|
||||
|
|
@ -737,7 +750,7 @@ class TestFormRequest(TestRequestBase):
|
|||
</form>"""
|
||||
)
|
||||
req = self.request_class.from_response(res)
|
||||
fs = _qs(req, to_unicode=True)
|
||||
fs = _qs_unicode(req)
|
||||
assert fs == {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]}
|
||||
|
||||
def test_from_response_radio(self):
|
||||
|
|
@ -1022,7 +1035,7 @@ class TestFormRequest(TestRequestBase):
|
|||
with pytest.raises(
|
||||
ValueError, match="formdata should be a dict or iterable of tuples"
|
||||
):
|
||||
FormRequest.from_response(response, formdata=123)
|
||||
FormRequest.from_response(response, formdata=123) # type: ignore[arg-type]
|
||||
|
||||
def test_form_response_with_custom_invalid_formdata_value_error(self):
|
||||
"""Test that a ValueError is raised for fault-inducing iterable formdata input"""
|
||||
|
|
@ -1037,7 +1050,7 @@ class TestFormRequest(TestRequestBase):
|
|||
with pytest.raises(
|
||||
ValueError, match="formdata should be a dict or iterable of tuples"
|
||||
):
|
||||
FormRequest.from_response(response, formdata=("a",))
|
||||
FormRequest.from_response(response, formdata=("a",)) # type: ignore[arg-type]
|
||||
|
||||
def test_get_form_with_xpath_no_form_parent(self):
|
||||
"""Test that _get_from raised a ValueError when an XPath selects an element
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import codecs
|
||||
from typing import cast
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -14,6 +15,12 @@ from tests.utils.bases.http_response import TestResponseBase
|
|||
class TestTextResponse(TestResponseBase):
|
||||
response_class = TextResponse
|
||||
|
||||
def _links_response(self) -> TextResponse:
|
||||
return cast("TextResponse", super()._links_response())
|
||||
|
||||
def _links_response_no_href(self) -> TextResponse:
|
||||
return cast("TextResponse", super()._links_response_no_href())
|
||||
|
||||
def test_follow_None_encoding(self):
|
||||
# unlike the base Response, TextResponse.follow() falls back to the
|
||||
# response encoding when encoding is None instead of raising
|
||||
|
|
@ -21,7 +28,7 @@ class TestTextResponse(TestResponseBase):
|
|||
req = r.follow("foo", encoding=None)
|
||||
assert req.encoding == "cp1252"
|
||||
|
||||
def test_replace(self):
|
||||
def test_replace(self) -> None:
|
||||
super().test_replace()
|
||||
r1 = self.response_class(
|
||||
"http://www.example.com", body="hello", encoding="cp852"
|
||||
|
|
@ -344,7 +351,7 @@ class TestTextResponse(TestResponseBase):
|
|||
def test_follow_selector_list(self):
|
||||
resp = self._links_response()
|
||||
with pytest.raises(ValueError, match="SelectorList"):
|
||||
resp.follow(resp.css("a"))
|
||||
resp.follow(resp.css("a")) # type: ignore[arg-type]
|
||||
|
||||
def test_follow_selector_invalid(self):
|
||||
resp = self._links_response()
|
||||
|
|
@ -616,7 +623,7 @@ class CustomResponse(TextResponse):
|
|||
class TestCustomResponse(TestTextResponse):
|
||||
response_class = CustomResponse
|
||||
|
||||
def test_copy(self):
|
||||
def test_copy(self) -> None:
|
||||
super().test_copy()
|
||||
r1 = self.response_class(
|
||||
url="https://example.org",
|
||||
|
|
@ -632,7 +639,7 @@ class TestCustomResponse(TestTextResponse):
|
|||
assert r1.lost == "lost"
|
||||
assert r2.lost is None
|
||||
|
||||
def test_replace(self):
|
||||
def test_replace(self) -> None:
|
||||
super().test_replace()
|
||||
r1 = self.response_class(
|
||||
url="https://example.org",
|
||||
|
|
|
|||
|
|
@ -85,6 +85,7 @@ class TestCrawl:
|
|||
url = self.mockserver.url("/status?n=200")
|
||||
crawler = get_crawler(SingleRequestSpider)
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
assert isinstance(crawler.spider, SingleRequestSpider)
|
||||
response = crawler.spider.meta["responses"][0]
|
||||
assert response.request.url == url
|
||||
|
||||
|
|
@ -94,6 +95,7 @@ class TestCrawl:
|
|||
url = self.mockserver.url(f"/status?n={status}")
|
||||
crawler = get_crawler(SingleRequestSpider)
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
assert isinstance(crawler.spider, SingleRequestSpider)
|
||||
failure = crawler.spider.meta["failure"]
|
||||
response = failure.value.response
|
||||
assert failure.request.url == url
|
||||
|
|
@ -111,6 +113,7 @@ class TestCrawl:
|
|||
},
|
||||
)
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
assert isinstance(crawler.spider, SingleRequestSpider)
|
||||
failure = crawler.spider.meta["failure"]
|
||||
assert failure.request.url == url
|
||||
assert isinstance(failure.value, ZeroDivisionError)
|
||||
|
|
@ -178,6 +181,7 @@ class TestCrawl:
|
|||
},
|
||||
)
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
assert isinstance(crawler.spider, SingleRequestSpider)
|
||||
response = crawler.spider.meta["responses"][0]
|
||||
assert response.body == b"Caught ZeroDivisionError"
|
||||
assert response.request.url == OVERRIDDEN_URL
|
||||
|
|
@ -201,6 +205,7 @@ class TestCrawl:
|
|||
},
|
||||
)
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
assert isinstance(crawler.spider, SingleRequestSpider)
|
||||
response = crawler.spider.meta["responses"][0]
|
||||
assert response.body == b"Caught ZeroDivisionError"
|
||||
assert response.request.url == url
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ class InjectArgumentsSpiderMiddleware:
|
|||
async for element in result:
|
||||
if (
|
||||
isinstance(element, Request)
|
||||
and element.callback
|
||||
and element.callback.__name__ == "parse_spider_mw_2"
|
||||
):
|
||||
element.cb_kwargs["from_process_spider_output"] = True
|
||||
|
|
@ -68,7 +69,12 @@ class KeywordArgumentsSpider(MockServerSpider):
|
|||
|
||||
checks: list[bool] = []
|
||||
|
||||
def _inc_checks(self, count: int = 1) -> None:
|
||||
assert self.crawler.stats
|
||||
self.crawler.stats.inc_value("boolean_checks", count)
|
||||
|
||||
async def start(self):
|
||||
assert self.mockserver
|
||||
data = {"key": "value", "number": 123, "callback": "some_callback"}
|
||||
yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data)
|
||||
yield Request(
|
||||
|
|
@ -89,9 +95,10 @@ class KeywordArgumentsSpider(MockServerSpider):
|
|||
yield Request(self.mockserver.url("/spider_mw"), self.parse_spider_mw)
|
||||
|
||||
def parse_first(self, response, key, number):
|
||||
assert self.mockserver
|
||||
self.checks.append(key == "value")
|
||||
self.checks.append(number == 123)
|
||||
self.crawler.stats.inc_value("boolean_checks", 2)
|
||||
self._inc_checks(2)
|
||||
yield response.follow(
|
||||
self.mockserver.url("/two"),
|
||||
self.parse_second,
|
||||
|
|
@ -100,28 +107,28 @@ class KeywordArgumentsSpider(MockServerSpider):
|
|||
|
||||
def parse_second(self, response, new_key):
|
||||
self.checks.append(new_key == "new_value")
|
||||
self.crawler.stats.inc_value("boolean_checks")
|
||||
self._inc_checks()
|
||||
|
||||
def parse_general(self, response, **kwargs):
|
||||
if response.url.endswith("/general_with"):
|
||||
self.checks.append(kwargs["key"] == "value")
|
||||
self.checks.append(kwargs["number"] == 123)
|
||||
self.checks.append(kwargs["callback"] == "some_callback")
|
||||
self.crawler.stats.inc_value("boolean_checks", 3)
|
||||
self._inc_checks(3)
|
||||
elif response.url.endswith("/general_without"):
|
||||
self.checks.append(kwargs == {})
|
||||
self.crawler.stats.inc_value("boolean_checks")
|
||||
self._inc_checks()
|
||||
|
||||
def parse_no_kwargs(self, response):
|
||||
self.checks.append(response.url.endswith("/no_kwargs"))
|
||||
self.crawler.stats.inc_value("boolean_checks")
|
||||
self._inc_checks()
|
||||
|
||||
def parse_default(self, response, key, number=None, default=99):
|
||||
self.checks.append(response.url.endswith("/default"))
|
||||
self.checks.append(key == "value")
|
||||
self.checks.append(number == 123)
|
||||
self.checks.append(default == 99)
|
||||
self.crawler.stats.inc_value("boolean_checks", 4)
|
||||
self._inc_checks(4)
|
||||
|
||||
def parse_takes_less(self, response, key, callback):
|
||||
"""
|
||||
|
|
@ -140,17 +147,18 @@ class KeywordArgumentsSpider(MockServerSpider):
|
|||
):
|
||||
self.checks.append(bool(from_process_request))
|
||||
self.checks.append(bool(from_process_response))
|
||||
self.crawler.stats.inc_value("boolean_checks", 2)
|
||||
self._inc_checks(2)
|
||||
|
||||
def parse_spider_mw(self, response, from_process_spider_input, from_process_start):
|
||||
assert self.mockserver
|
||||
self.checks.append(bool(from_process_spider_input))
|
||||
self.checks.append(bool(from_process_start))
|
||||
self.crawler.stats.inc_value("boolean_checks", 2)
|
||||
self._inc_checks(2)
|
||||
return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2)
|
||||
|
||||
def parse_spider_mw_2(self, response, from_process_spider_output):
|
||||
self.checks.append(bool(from_process_spider_output))
|
||||
self.crawler.stats.inc_value("boolean_checks", 1)
|
||||
self._inc_checks()
|
||||
|
||||
|
||||
class TestCallbackKeywordArguments:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy import Request, Spider
|
||||
from scrapy.http import JsonRequest
|
||||
from scrapy.http import JsonRequest, Response
|
||||
from scrapy.utils.request import request_from_dict
|
||||
|
||||
|
||||
|
|
@ -10,7 +13,7 @@ class CustomRequest(Request):
|
|||
|
||||
|
||||
class TestRequestSerialization:
|
||||
def setup_method(self):
|
||||
def setup_method(self) -> None:
|
||||
self.spider = MethodsSpider()
|
||||
|
||||
def test_basic(self):
|
||||
|
|
@ -42,12 +45,14 @@ class TestRequestSerialization:
|
|||
r = Request("http://www.example.com", body=b"\xc2\xa3")
|
||||
self._assert_serializes_ok(r)
|
||||
|
||||
def _assert_serializes_ok(self, request, spider=None):
|
||||
def _assert_serializes_ok(
|
||||
self, request: Request, spider: Spider | None = None
|
||||
) -> None:
|
||||
d = request.to_dict(spider=spider)
|
||||
request2 = request_from_dict(d, spider=spider)
|
||||
self._assert_same_request(request, request2)
|
||||
|
||||
def _assert_same_request(self, r1, r2):
|
||||
def _assert_same_request(self, r1: Request, r2: Request) -> None:
|
||||
assert r1.__class__ == r2.__class__
|
||||
assert r1.url == r2.url
|
||||
assert r1.callback == r2.callback
|
||||
|
|
@ -64,6 +69,7 @@ class TestRequestSerialization:
|
|||
assert r1.dont_filter == r2.dont_filter
|
||||
assert r1.flags == r2.flags
|
||||
if isinstance(r1, JsonRequest):
|
||||
assert isinstance(r2, JsonRequest)
|
||||
assert r1.dumps_kwargs == r2.dumps_kwargs
|
||||
|
||||
def test_request_class(self):
|
||||
|
|
@ -83,8 +89,8 @@ class TestRequestSerialization:
|
|||
def test_reference_callback_serialization(self):
|
||||
r = Request(
|
||||
"http://www.example.com",
|
||||
callback=self.spider.parse_item_reference,
|
||||
errback=self.spider.handle_error_reference,
|
||||
callback=self.spider.parse_item_reference, # type: ignore[arg-type,misc]
|
||||
errback=self.spider.handle_error_reference, # type: ignore[arg-type,misc]
|
||||
)
|
||||
self._assert_serializes_ok(r, spider=self.spider)
|
||||
request_dict = r.to_dict(spider=self.spider)
|
||||
|
|
@ -94,8 +100,8 @@ class TestRequestSerialization:
|
|||
def test_private_reference_callback_serialization(self):
|
||||
r = Request(
|
||||
"http://www.example.com",
|
||||
callback=self.spider._MethodsSpider__parse_item_reference,
|
||||
errback=self.spider._MethodsSpider__handle_error_reference,
|
||||
callback=self.spider._MethodsSpider__parse_item_reference, # type: ignore[attr-defined]
|
||||
errback=self.spider._MethodsSpider__handle_error_reference, # type: ignore[attr-defined]
|
||||
)
|
||||
self._assert_serializes_ok(r, spider=self.spider)
|
||||
request_dict = r.to_dict(spider=self.spider)
|
||||
|
|
@ -105,7 +111,7 @@ class TestRequestSerialization:
|
|||
def test_private_callback_serialization(self):
|
||||
r = Request(
|
||||
"http://www.example.com",
|
||||
callback=self.spider._MethodsSpider__parse_item_private,
|
||||
callback=self.spider._MethodsSpider__parse_item_private, # type: ignore[attr-defined]
|
||||
errback=self.spider.handle_error,
|
||||
)
|
||||
self._assert_serializes_ok(r, spider=self.spider)
|
||||
|
|
@ -113,7 +119,7 @@ class TestRequestSerialization:
|
|||
def test_mixin_private_callback_serialization(self):
|
||||
r = Request(
|
||||
"http://www.example.com",
|
||||
callback=self.spider._SpiderMixin__mixin_callback,
|
||||
callback=self.spider._SpiderMixin__mixin_callback, # type: ignore[attr-defined]
|
||||
errback=self.spider.handle_error,
|
||||
)
|
||||
self._assert_serializes_ok(r, spider=self.spider)
|
||||
|
|
@ -127,7 +133,7 @@ class TestRequestSerialization:
|
|||
self._assert_serializes_ok(r, spider=self.spider)
|
||||
|
||||
def test_unserializable_callback1(self):
|
||||
r = Request("http://www.example.com", callback=lambda x: x)
|
||||
r = Request("http://www.example.com", callback=lambda x: x) # type: ignore[misc]
|
||||
with pytest.raises(
|
||||
ValueError, match="is not an instance method in: <MethodsSpider"
|
||||
):
|
||||
|
|
@ -144,12 +150,12 @@ class TestRequestSerialization:
|
|||
class MySpider(Spider):
|
||||
name = "my_spider"
|
||||
|
||||
def parse(self, response):
|
||||
def parse(self, response: Response) -> None:
|
||||
pass
|
||||
|
||||
spider = MySpider()
|
||||
r = Request("http://www.example.com", callback=spider.parse)
|
||||
spider.parse = None
|
||||
spider.parse = None # type: ignore[method-assign,assignment]
|
||||
with pytest.raises(ValueError, match="is not an instance method in: <MySpider"):
|
||||
r.to_dict(spider=spider)
|
||||
|
||||
|
|
@ -157,7 +163,7 @@ class TestRequestSerialization:
|
|||
"""Callback method is not available in the spider passed to from_dict"""
|
||||
spider = SpiderDelegation()
|
||||
r = Request("http://www.example.com", callback=spider.delegated_callback)
|
||||
d = r.to_dict(spider=spider)
|
||||
d = r.to_dict(spider=spider) # type: ignore[arg-type]
|
||||
with pytest.raises(
|
||||
ValueError, match="Method 'delegated_callback' not found in: <Spider"
|
||||
):
|
||||
|
|
@ -165,28 +171,30 @@ class TestRequestSerialization:
|
|||
|
||||
|
||||
class SpiderMixin:
|
||||
def __mixin_callback(self, response): # pylint: disable=unused-private-member
|
||||
def __mixin_callback( # pylint: disable=unused-private-member
|
||||
self, response: Response
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
|
||||
class SpiderDelegation:
|
||||
def delegated_callback(self, response):
|
||||
def delegated_callback(self, response: Response) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def parse_item(response):
|
||||
def parse_item(response: Response) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def handle_error(failure):
|
||||
def handle_error(failure: Failure) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def private_parse_item(response):
|
||||
def private_parse_item(response: Response) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def private_handle_error(failure):
|
||||
def private_handle_error(failure: Failure) -> None:
|
||||
pass
|
||||
|
||||
|
||||
|
|
@ -197,15 +205,17 @@ class MethodsSpider(Spider, SpiderMixin):
|
|||
__parse_item_reference = private_parse_item
|
||||
__handle_error_reference = private_handle_error
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
def __init__(self, **kwargs: Any) -> None:
|
||||
super().__init__(**kwargs)
|
||||
self.delegated_callback = SpiderDelegation().delegated_callback
|
||||
|
||||
def parse_item(self, response):
|
||||
def parse_item(self, response: Response) -> None:
|
||||
pass
|
||||
|
||||
def handle_error(self, failure):
|
||||
def handle_error(self, failure: Failure) -> None:
|
||||
pass
|
||||
|
||||
def __parse_item_private(self, response): # pylint: disable=unused-private-member
|
||||
def __parse_item_private( # pylint: disable=unused-private-member
|
||||
self, response: Response
|
||||
) -> None:
|
||||
pass
|
||||
|
|
|
|||
|
|
@ -1,57 +1,62 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from scrapy.signals import request_left_downloader
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.test import get_crawler
|
||||
from tests.mockserver.http import MockServer
|
||||
from tests.utils.decorators import inline_callbacks_test
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from scrapy import Request
|
||||
from scrapy.crawler import Crawler
|
||||
from tests.mockserver.http import MockServer
|
||||
|
||||
|
||||
class SignalCatcherSpider(Spider):
|
||||
name = "signal_catcher"
|
||||
|
||||
def __init__(self, crawler, url, *args, **kwargs):
|
||||
def __init__(self, crawler: Crawler, url: str, *args: Any, **kwargs: Any):
|
||||
super().__init__(*args, **kwargs)
|
||||
crawler.signals.connect(self.on_request_left, signal=request_left_downloader)
|
||||
self.caught_times = 0
|
||||
self.start_urls = [url]
|
||||
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, *args, **kwargs):
|
||||
def from_crawler(
|
||||
cls, crawler: Crawler, *args: Any, **kwargs: Any
|
||||
) -> SignalCatcherSpider:
|
||||
return cls(crawler, *args, **kwargs)
|
||||
|
||||
def on_request_left(self, request, spider):
|
||||
def on_request_left(self, request: Request, spider: Spider) -> None:
|
||||
self.caught_times += 1
|
||||
|
||||
|
||||
class TestCatching:
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
cls.mockserver = MockServer()
|
||||
cls.mockserver.__enter__()
|
||||
|
||||
@classmethod
|
||||
def teardown_class(cls):
|
||||
cls.mockserver.__exit__(None, None, None)
|
||||
|
||||
@inline_callbacks_test
|
||||
def test_success(self):
|
||||
def test_success(self, mockserver: MockServer):
|
||||
crawler = get_crawler(SignalCatcherSpider)
|
||||
yield crawler.crawl(self.mockserver.url("/status?n=200"))
|
||||
yield crawler.crawl(mockserver.url("/status?n=200"))
|
||||
assert isinstance(crawler.spider, SignalCatcherSpider)
|
||||
assert crawler.spider.caught_times == 1
|
||||
|
||||
@inline_callbacks_test
|
||||
def test_timeout(self):
|
||||
def test_timeout(self, mockserver: MockServer):
|
||||
crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})
|
||||
yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
|
||||
yield crawler.crawl(mockserver.url("/delay?n=0.2"))
|
||||
assert isinstance(crawler.spider, SignalCatcherSpider)
|
||||
assert crawler.spider.caught_times == 1
|
||||
|
||||
@inline_callbacks_test
|
||||
def test_disconnect(self):
|
||||
def test_disconnect(self, mockserver: MockServer):
|
||||
crawler = get_crawler(SignalCatcherSpider)
|
||||
yield crawler.crawl(self.mockserver.url("/drop"))
|
||||
yield crawler.crawl(mockserver.url("/drop"))
|
||||
assert isinstance(crawler.spider, SignalCatcherSpider)
|
||||
assert crawler.spider.caught_times == 1
|
||||
|
||||
@inline_callbacks_test
|
||||
def test_noconnect(self):
|
||||
crawler = get_crawler(SignalCatcherSpider)
|
||||
yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")
|
||||
assert isinstance(crawler.spider, SignalCatcherSpider)
|
||||
assert crawler.spider.caught_times == 1
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
|
||||
from scrapy.robotstxt import (
|
||||
|
|
@ -10,22 +14,32 @@ from scrapy.robotstxt import (
|
|||
from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
|
||||
from tests.utils.robotstxt import rerp_available
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# typing.Self requires Python 3.11
|
||||
from typing_extensions import Self
|
||||
|
||||
from scrapy.crawler import Crawler
|
||||
|
||||
|
||||
class BaseRobotParserTest:
|
||||
def _setUp(self, parser_cls):
|
||||
parser_cls: type[RobotParser]
|
||||
|
||||
def _setUp(self, parser_cls: type[RobotParser]) -> None:
|
||||
self.parser_cls = parser_cls
|
||||
|
||||
def _parse(self, robotstxt_body: bytes) -> RobotParser:
|
||||
# The parser backends only use the crawler to get the spider to log with.
|
||||
return self.parser_cls.from_crawler(None, robotstxt_body) # type: ignore[arg-type]
|
||||
|
||||
def test_allowed(self):
|
||||
robotstxt_robotstxt_body = (
|
||||
b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10"
|
||||
)
|
||||
rp = self.parser_cls.from_crawler(
|
||||
crawler=None, robotstxt_body=robotstxt_robotstxt_body
|
||||
)
|
||||
rp = self._parse(robotstxt_robotstxt_body)
|
||||
assert rp.allowed("https://www.site.local/allowed", "*")
|
||||
assert not rp.allowed("https://www.site.local/disallowed", "*")
|
||||
|
||||
def test_allowed_wildcards(self):
|
||||
def test_allowed_wildcards(self) -> None:
|
||||
robotstxt_robotstxt_body = b"""User-agent: first
|
||||
Disallow: /disallowed/*/end$
|
||||
|
||||
|
|
@ -33,9 +47,7 @@ class BaseRobotParserTest:
|
|||
Allow: /*allowed
|
||||
Disallow: /
|
||||
"""
|
||||
rp = self.parser_cls.from_crawler(
|
||||
crawler=None, robotstxt_body=robotstxt_robotstxt_body
|
||||
)
|
||||
rp = self._parse(robotstxt_robotstxt_body)
|
||||
|
||||
assert rp.allowed("https://www.site.local/disallowed", "first")
|
||||
assert not rp.allowed("https://www.site.local/disallowed/xyz/end", "first")
|
||||
|
|
@ -46,23 +58,19 @@ class BaseRobotParserTest:
|
|||
assert rp.allowed("https://www.site.local/is_still_allowed", "second")
|
||||
assert rp.allowed("https://www.site.local/is_allowed_too", "second")
|
||||
|
||||
def test_length_based_precedence(self):
|
||||
def test_length_based_precedence(self) -> None:
|
||||
robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"
|
||||
rp = self.parser_cls.from_crawler(
|
||||
crawler=None, robotstxt_body=robotstxt_robotstxt_body
|
||||
)
|
||||
rp = self._parse(robotstxt_robotstxt_body)
|
||||
assert rp.allowed("https://www.site.local/page", "*")
|
||||
|
||||
def test_order_based_precedence(self):
|
||||
def test_order_based_precedence(self) -> None:
|
||||
robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"
|
||||
rp = self.parser_cls.from_crawler(
|
||||
crawler=None, robotstxt_body=robotstxt_robotstxt_body
|
||||
)
|
||||
rp = self._parse(robotstxt_robotstxt_body)
|
||||
assert not rp.allowed("https://www.site.local/page", "*")
|
||||
|
||||
def test_empty_response(self):
|
||||
"""empty response should equal 'allow all'"""
|
||||
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b"")
|
||||
rp = self._parse(b"")
|
||||
assert rp.allowed("https://site.local/", "*")
|
||||
assert rp.allowed("https://site.local/", "chrome")
|
||||
assert rp.allowed("https://site.local/index.html", "*")
|
||||
|
|
@ -71,9 +79,7 @@ class BaseRobotParserTest:
|
|||
def test_garbage_response(self):
|
||||
"""garbage response should be discarded, equal 'allow all'"""
|
||||
robotstxt_robotstxt_body = b"GIF89a\xd3\x00\xfe\x00\xa2"
|
||||
rp = self.parser_cls.from_crawler(
|
||||
crawler=None, robotstxt_body=robotstxt_robotstxt_body
|
||||
)
|
||||
rp = self._parse(robotstxt_robotstxt_body)
|
||||
assert rp.allowed("https://site.local/", "*")
|
||||
assert rp.allowed("https://site.local/", "chrome")
|
||||
assert rp.allowed("https://site.local/index.html", "*")
|
||||
|
|
@ -81,12 +87,12 @@ class BaseRobotParserTest:
|
|||
|
||||
def test_crawl_delay(self):
|
||||
robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n"
|
||||
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
|
||||
rp = self._parse(robotstxt_body)
|
||||
assert rp.crawl_delay("*") == 10.0
|
||||
|
||||
def test_crawl_delay_unset(self):
|
||||
robotstxt_body = b"User-agent: *\nDisallow: /private\n"
|
||||
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
|
||||
rp = self._parse(robotstxt_body)
|
||||
assert rp.crawl_delay("*") is None
|
||||
|
||||
def test_unicode_url_and_useragent(self):
|
||||
|
|
@ -100,9 +106,7 @@ class BaseRobotParserTest:
|
|||
|
||||
User-Agent: UnicödeBöt
|
||||
Disallow: /some/randome/page.html""".encode()
|
||||
rp = self.parser_cls.from_crawler(
|
||||
crawler=None, robotstxt_body=robotstxt_robotstxt_body
|
||||
)
|
||||
rp = self._parse(robotstxt_robotstxt_body)
|
||||
assert rp.allowed("https://site.local/", "*")
|
||||
assert not rp.allowed("https://site.local/admin/", "*")
|
||||
assert not rp.allowed("https://site.local/static/", "*")
|
||||
|
|
@ -117,15 +121,13 @@ class TestRobotParser:
|
|||
def test_crawl_delay_unsupported(self):
|
||||
class AllowAllRobotParser(RobotParser):
|
||||
@classmethod
|
||||
def from_crawler(cls, crawler, robotstxt_body):
|
||||
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
|
||||
return cls()
|
||||
|
||||
def allowed(self, url, user_agent):
|
||||
def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
|
||||
return True
|
||||
|
||||
rp = AllowAllRobotParser.from_crawler(
|
||||
crawler=None, robotstxt_body=b"User-agent: *\nCrawl-delay: 10\n"
|
||||
)
|
||||
rp = AllowAllRobotParser()
|
||||
assert rp.crawl_delay("*") is None
|
||||
|
||||
|
||||
|
|
@ -162,21 +164,21 @@ class TestPythonRobotParser(BaseRobotParserTest):
|
|||
not STDLIB_IMPROVED_ROBOTFILEPARSER,
|
||||
reason="RobotFileParser from this Python version does not support length based directives precedence.",
|
||||
)
|
||||
def test_length_based_precedence(self):
|
||||
def test_length_based_precedence(self) -> None:
|
||||
super().test_length_based_precedence()
|
||||
|
||||
@pytest.mark.skipif(
|
||||
STDLIB_IMPROVED_ROBOTFILEPARSER,
|
||||
reason="RobotFileParser from this Python version does not support order based directives precedence.",
|
||||
)
|
||||
def test_order_based_precedence(self):
|
||||
def test_order_based_precedence(self) -> None:
|
||||
super().test_order_based_precedence()
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not STDLIB_IMPROVED_ROBOTFILEPARSER,
|
||||
reason="RobotFileParser from this Python version does not support wildcards.",
|
||||
)
|
||||
def test_allowed_wildcards(self):
|
||||
def test_allowed_wildcards(self) -> None:
|
||||
super().test_allowed_wildcards()
|
||||
|
||||
|
||||
|
|
@ -185,7 +187,7 @@ class TestRerpRobotParser(BaseRobotParserTest):
|
|||
def setup_method(self):
|
||||
super()._setUp(RerpRobotParser)
|
||||
|
||||
def test_length_based_precedence(self):
|
||||
def test_length_based_precedence(self) -> None:
|
||||
pytest.skip("Rerp does not support length based directives precedence.")
|
||||
|
||||
|
||||
|
|
@ -193,5 +195,5 @@ class TestProtegoRobotParser(BaseRobotParserTest):
|
|||
def setup_method(self):
|
||||
super()._setUp(ProtegoRobotParser)
|
||||
|
||||
def test_order_based_precedence(self):
|
||||
def test_order_based_precedence(self) -> None:
|
||||
pytest.skip("Protego does not support order based directives precedence.")
|
||||
|
|
|
|||
|
|
@ -12,7 +12,9 @@ from scrapy.utils.asyncgen import as_async_generator
|
|||
from scrapy.utils.asyncio import (
|
||||
AsyncioLoopingCall,
|
||||
_parallel_asyncio,
|
||||
call_later,
|
||||
is_asyncio_available,
|
||||
sleep,
|
||||
)
|
||||
from tests.utils.decorators import coroutine_test
|
||||
|
||||
|
|
@ -26,6 +28,15 @@ async def test_is_asyncio_available(reactor_pytest: str) -> None:
|
|||
assert is_asyncio_available() == (reactor_pytest != "default")
|
||||
|
||||
|
||||
@coroutine_test
|
||||
async def test_sleep() -> None:
|
||||
events: list[str] = []
|
||||
call_later(0.05, events.append, "call_later")
|
||||
await sleep(0.1)
|
||||
events.append("sleep")
|
||||
assert events == ["call_later", "sleep"]
|
||||
|
||||
|
||||
@pytest.mark.only_asyncio
|
||||
class TestParallelAsyncio:
|
||||
"""Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync."""
|
||||
|
|
|
|||
|
|
@ -1,10 +1,38 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from importlib.util import find_spec
|
||||
from io import BytesIO
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pytest
|
||||
from pexpect import EOF
|
||||
|
||||
from scrapy.utils.console import get_shell_embed_func, start_python_console
|
||||
from scrapy.utils.test import get_testenv
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from pathlib import Path
|
||||
|
||||
CONSOLE = """
|
||||
from scrapy.utils.console import start_python_console
|
||||
|
||||
start_python_console(banner="SHELL-READY", shells=["ipython"])
|
||||
"""
|
||||
|
||||
CONSOLE_IN_RUNNING_LOOP = """
|
||||
import asyncio
|
||||
|
||||
from scrapy.utils.console import start_python_console
|
||||
|
||||
|
||||
async def main():
|
||||
start_python_console(banner="SHELL-READY", shells=["ipython"])
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
"""
|
||||
|
||||
|
||||
def test_get_shell_embed_func():
|
||||
|
|
@ -61,6 +89,76 @@ def test_get_shell_embed_func_default():
|
|||
assert shell.__name__ == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(find_spec("IPython") is None, reason="IPython is not installed")
|
||||
class TestIPythonShell:
|
||||
"""Starting an IPython shell, with and without an asyncio event loop already
|
||||
running in the calling thread. The latter happens when inspect_response() is
|
||||
called from a spider callback while using the asyncio reactor."""
|
||||
|
||||
@staticmethod
|
||||
def _env(tmp_path: Path) -> dict[str, str]:
|
||||
env = get_testenv()
|
||||
# Keep IPython away from the profile and history of the user running the tests.
|
||||
env["IPYTHONDIR"] = str(tmp_path)
|
||||
return env
|
||||
|
||||
def test_simple_prompt(self, tmp_path: Path) -> None:
|
||||
"""IPython falls back to its simple prompt, which needs no event loop,
|
||||
when stdin is not a TTY."""
|
||||
env = self._env(tmp_path)
|
||||
p = subprocess.run(
|
||||
[sys.executable, "-c", CONSOLE_IN_RUNNING_LOOP],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
encoding="utf-8",
|
||||
timeout=60,
|
||||
env=env,
|
||||
stdin=subprocess.DEVNULL,
|
||||
)
|
||||
output = p.stdout + p.stderr
|
||||
assert "SHELL-READY" in output
|
||||
assert p.returncode == 0, output
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform == "win32", reason="requires a POSIX pseudo-terminal"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"script",
|
||||
[CONSOLE, CONSOLE_IN_RUNNING_LOOP],
|
||||
ids=["no_running_loop", "running_loop"],
|
||||
)
|
||||
def test_tty(self, tmp_path: Path, script: str) -> None:
|
||||
"""IPython uses prompt_toolkit, which needs an event loop of its own,
|
||||
when stdin is a TTY."""
|
||||
# pexpect only defines spawn, which needs a pseudo-terminal, on POSIX.
|
||||
from pexpect import spawn # noqa: PLC0415
|
||||
|
||||
env = self._env(tmp_path)
|
||||
env.pop("IPY_TEST_SIMPLE_PROMPT", None)
|
||||
env["TERM"] = "xterm"
|
||||
logfile = BytesIO()
|
||||
p = spawn(
|
||||
sys.executable,
|
||||
["-c", script],
|
||||
env=env,
|
||||
timeout=60,
|
||||
)
|
||||
p.logfile_read = logfile
|
||||
try:
|
||||
# Wait for the prompt, which prompt_toolkit draws once it is done
|
||||
# querying the terminal, before typing into it.
|
||||
p.expect(r"In \[")
|
||||
p.sendline("21*2")
|
||||
p.expect_exact("42")
|
||||
p.sendline("exit()")
|
||||
p.expect(EOF)
|
||||
finally:
|
||||
p.close()
|
||||
output = logfile.getvalue().decode()
|
||||
assert "Traceback" not in output
|
||||
assert p.exitstatus == 0, output
|
||||
|
||||
|
||||
def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def embed(namespace: dict[str, object], banner: str) -> None:
|
||||
raise SystemExit
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
|
@ -8,8 +7,6 @@ from typing import TYPE_CHECKING
|
|||
from twisted.internet.defer import Deferred
|
||||
|
||||
from scrapy.settings import Settings, default_settings
|
||||
from scrapy.utils.asyncio import is_asyncio_available
|
||||
from scrapy.utils.defer import maybe_deferred_to_future
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
|
@ -23,13 +20,6 @@ def twisted_sleep(seconds: float):
|
|||
return d
|
||||
|
||||
|
||||
async def async_sleep(seconds: float) -> None:
|
||||
if is_asyncio_available():
|
||||
await asyncio.sleep(seconds)
|
||||
else:
|
||||
await maybe_deferred_to_future(twisted_sleep(seconds))
|
||||
|
||||
|
||||
def get_script_run_env() -> dict[str, str]:
|
||||
"""Return a OS environment dict suitable to run scripts shipped with tests."""
|
||||
|
||||
|
|
|
|||
|
|
@ -3,8 +3,9 @@ from abc import ABC, abstractmethod
|
|||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from twisted.python.failure import Failure
|
||||
|
||||
from scrapy.http import Headers, Request
|
||||
from scrapy.http import Headers, Request, Response
|
||||
from scrapy.http.request import NO_CALLBACK
|
||||
from scrapy.utils.request import request_to_curl
|
||||
|
||||
|
|
@ -22,15 +23,15 @@ class TestRequestBase(ABC):
|
|||
def test_init(self):
|
||||
# Request requires url in the __init__ method
|
||||
with pytest.raises(TypeError):
|
||||
self.request_class()
|
||||
self.request_class() # type: ignore[call-arg]
|
||||
|
||||
# url argument must be basestring
|
||||
with pytest.raises(TypeError):
|
||||
self.request_class(123)
|
||||
self.request_class(123) # type: ignore[arg-type]
|
||||
|
||||
# priority argument must be an integer
|
||||
with pytest.raises(TypeError, match="Request priority not an integer"):
|
||||
self.request_class("http://www.example.com", priority="1")
|
||||
self.request_class("http://www.example.com", priority="1") # type: ignore[arg-type]
|
||||
|
||||
r = self.request_class("http://www.example.com")
|
||||
assert isinstance(r.url, str)
|
||||
|
|
@ -205,14 +206,17 @@ class TestRequestBase(ABC):
|
|||
def test_copy(self):
|
||||
"""Test Request copy"""
|
||||
|
||||
def somecallback():
|
||||
def somecallback(response: Response) -> None:
|
||||
pass
|
||||
|
||||
def someerrback(failure: Failure) -> None:
|
||||
pass
|
||||
|
||||
r1 = self.request_class(
|
||||
"http://www.example.com",
|
||||
flags=["f1", "f2"],
|
||||
callback=somecallback,
|
||||
errback=somecallback,
|
||||
errback=someerrback,
|
||||
)
|
||||
r1.meta["foo"] = "bar"
|
||||
r1.cb_kwargs["key"] = "value"
|
||||
|
|
@ -220,7 +224,7 @@ class TestRequestBase(ABC):
|
|||
|
||||
# make sure callbaclks are copied
|
||||
assert r1.callback is somecallback
|
||||
assert r1.errback is somecallback
|
||||
assert r1.errback is someerrback
|
||||
assert r2.callback is r1.callback
|
||||
assert r2.errback is r1.errback
|
||||
|
||||
|
|
@ -251,7 +255,7 @@ class TestRequestBase(ABC):
|
|||
def test_copy_inherited_classes(self):
|
||||
"""Test Request children copies preserve their class"""
|
||||
|
||||
class CustomRequest(self.request_class):
|
||||
class CustomRequest(self.request_class): # type: ignore[misc,name-defined]
|
||||
pass
|
||||
|
||||
r1 = CustomRequest("http://www.example.com")
|
||||
|
|
@ -283,7 +287,9 @@ class TestRequestBase(ABC):
|
|||
assert r4.dont_filter is False
|
||||
|
||||
# the cls argument allows changing the resulting class
|
||||
custom_request_cls = type("CustomRequest", (self.request_class,), {})
|
||||
custom_request_cls: type[Request] = type(
|
||||
"CustomRequest", (self.request_class,), {}
|
||||
)
|
||||
r5 = r1.replace(cls=custom_request_cls)
|
||||
assert isinstance(r5, custom_request_cls)
|
||||
assert r5.url == r1.url
|
||||
|
|
@ -295,33 +301,36 @@ class TestRequestBase(ABC):
|
|||
def test_immutable_attributes(self):
|
||||
r = self.request_class("http://example.com")
|
||||
with pytest.raises(AttributeError):
|
||||
r.url = "http://example2.com"
|
||||
r.url = "http://example2.com" # type: ignore[misc]
|
||||
with pytest.raises(AttributeError):
|
||||
r.body = "xxx"
|
||||
r.body = "xxx" # type: ignore[misc,assignment]
|
||||
|
||||
def test_callback_and_errback(self):
|
||||
def a_function():
|
||||
def a_callback(response: Response) -> None:
|
||||
pass
|
||||
|
||||
def an_errback(failure: Failure) -> None:
|
||||
pass
|
||||
|
||||
r1 = self.request_class("http://example.com")
|
||||
assert r1.callback is None
|
||||
assert r1.errback is None
|
||||
|
||||
r2 = self.request_class("http://example.com", callback=a_function)
|
||||
assert r2.callback is a_function
|
||||
r2 = self.request_class("http://example.com", callback=a_callback)
|
||||
assert r2.callback is a_callback
|
||||
assert r2.errback is None
|
||||
|
||||
r3 = self.request_class("http://example.com", errback=a_function)
|
||||
r3 = self.request_class("http://example.com", errback=an_errback)
|
||||
assert r3.callback is None
|
||||
assert r3.errback is a_function
|
||||
assert r3.errback is an_errback
|
||||
|
||||
r4 = self.request_class(
|
||||
url="http://example.com",
|
||||
callback=a_function,
|
||||
errback=a_function,
|
||||
callback=a_callback,
|
||||
errback=an_errback,
|
||||
)
|
||||
assert r4.callback is a_function
|
||||
assert r4.errback is a_function
|
||||
assert r4.callback is a_callback
|
||||
assert r4.errback is an_errback
|
||||
|
||||
r5 = self.request_class(
|
||||
url="http://example.com",
|
||||
|
|
@ -329,18 +338,18 @@ class TestRequestBase(ABC):
|
|||
errback=NO_CALLBACK,
|
||||
)
|
||||
assert r5.callback is NO_CALLBACK
|
||||
assert r5.errback is NO_CALLBACK
|
||||
assert r5.errback is NO_CALLBACK # type: ignore[comparison-overlap]
|
||||
|
||||
def test_callback_and_errback_type(self):
|
||||
with pytest.raises(TypeError):
|
||||
self.request_class("http://example.com", callback="a_function")
|
||||
self.request_class("http://example.com", callback="a_function") # type: ignore[arg-type]
|
||||
with pytest.raises(TypeError):
|
||||
self.request_class("http://example.com", errback="a_function")
|
||||
self.request_class("http://example.com", errback="a_function") # type: ignore[arg-type]
|
||||
with pytest.raises(TypeError):
|
||||
self.request_class(
|
||||
url="http://example.com",
|
||||
callback="a_function",
|
||||
errback="a_function",
|
||||
callback="a_function", # type: ignore[arg-type]
|
||||
errback="a_function", # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
def test_setters(self):
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import pytest
|
|||
from w3lib.encoding import resolve_encoding
|
||||
|
||||
from scrapy.exceptions import NotSupported
|
||||
from scrapy.http import Headers, Request, Response
|
||||
from scrapy.http import Headers, Request, Response, TextResponse
|
||||
from scrapy.link import Link
|
||||
from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS
|
||||
from tests import get_testdata
|
||||
|
|
@ -15,6 +15,8 @@ from tests import get_testdata
|
|||
if TYPE_CHECKING:
|
||||
from collections.abc import Iterable
|
||||
|
||||
from parsel import Selector
|
||||
|
||||
|
||||
class TestResponseBase(ABC):
|
||||
@property
|
||||
|
|
@ -25,14 +27,14 @@ class TestResponseBase(ABC):
|
|||
def test_init(self):
|
||||
# Response requires url in the constructor
|
||||
with pytest.raises(TypeError):
|
||||
self.response_class()
|
||||
self.response_class() # type: ignore[call-arg]
|
||||
assert isinstance(
|
||||
self.response_class("http://example.com/"), self.response_class
|
||||
)
|
||||
with pytest.raises(TypeError):
|
||||
self.response_class(b"http://example.com")
|
||||
self.response_class(b"http://example.com") # type: ignore[arg-type]
|
||||
with pytest.raises(TypeError):
|
||||
self.response_class(url="http://example.com", body={})
|
||||
self.response_class(url="http://example.com", body={}) # type: ignore[arg-type]
|
||||
# body can be str or None
|
||||
assert isinstance(
|
||||
self.response_class("http://example.com/", body=b""),
|
||||
|
|
@ -67,12 +69,12 @@ class TestResponseBase(ABC):
|
|||
|
||||
r = self.response_class("http://www.example.com", status=301)
|
||||
assert r.status == 301
|
||||
r = self.response_class("http://www.example.com", status="301")
|
||||
r = self.response_class("http://www.example.com", status="301") # type: ignore[arg-type]
|
||||
assert r.status == 301
|
||||
with pytest.raises(ValueError, match=r"invalid literal for int\(\)"):
|
||||
self.response_class("http://example.com", status="lala200")
|
||||
self.response_class("http://example.com", status="lala200") # type: ignore[arg-type]
|
||||
|
||||
def test_copy(self):
|
||||
def test_copy(self) -> None:
|
||||
"""Test Response copy"""
|
||||
|
||||
r1 = self.response_class("http://www.example.com", body=b"Some body")
|
||||
|
|
@ -121,7 +123,7 @@ class TestResponseBase(ABC):
|
|||
def test_copy_inherited_classes(self):
|
||||
"""Test Response children copies preserve their class"""
|
||||
|
||||
class CustomResponse(self.response_class):
|
||||
class CustomResponse(self.response_class): # type: ignore[misc,name-defined]
|
||||
pass
|
||||
|
||||
r1 = CustomResponse("http://www.example.com")
|
||||
|
|
@ -129,7 +131,7 @@ class TestResponseBase(ABC):
|
|||
|
||||
assert isinstance(r2, CustomResponse)
|
||||
|
||||
def test_replace(self):
|
||||
def test_replace(self) -> None:
|
||||
"""Test Response.replace() method"""
|
||||
hdrs = Headers({"key": "value"})
|
||||
r1 = self.response_class("http://www.example.com")
|
||||
|
|
@ -146,7 +148,9 @@ class TestResponseBase(ABC):
|
|||
assert r4.body == b""
|
||||
assert not r4.flags
|
||||
|
||||
def _assert_response_values(self, response, encoding, body):
|
||||
def _assert_response_values(
|
||||
self, response: TextResponse, encoding: str, body: str | bytes
|
||||
) -> None:
|
||||
if isinstance(body, str):
|
||||
body_unicode = body
|
||||
body_bytes = body.encode(encoding)
|
||||
|
|
@ -160,15 +164,15 @@ class TestResponseBase(ABC):
|
|||
assert response.body == body_bytes
|
||||
assert response.text == body_unicode
|
||||
|
||||
def _assert_response_encoding(self, response, encoding):
|
||||
def _assert_response_encoding(self, response: TextResponse, encoding: str) -> None:
|
||||
assert response.encoding == resolve_encoding(encoding)
|
||||
|
||||
def test_immutable_attributes(self):
|
||||
r = self.response_class("http://example.com")
|
||||
with pytest.raises(AttributeError):
|
||||
r.url = "http://example2.com"
|
||||
r.url = "http://example2.com" # type: ignore[misc]
|
||||
with pytest.raises(AttributeError):
|
||||
r.body = "xxx"
|
||||
r.body = "xxx" # type: ignore[misc,assignment]
|
||||
|
||||
def test_setter_mutable_lazy_loading(self):
|
||||
"""Mutable attributes are set internally to None only until they are
|
||||
|
|
@ -256,7 +260,7 @@ class TestResponseBase(ABC):
|
|||
def test_follow_None_url(self):
|
||||
r = self.response_class("http://example.com")
|
||||
with pytest.raises(ValueError, match="url can't be None"):
|
||||
r.follow(None)
|
||||
r.follow(None) # type: ignore[arg-type]
|
||||
|
||||
def test_follow_None_encoding(self):
|
||||
r = self.response_class("http://example.com")
|
||||
|
|
@ -325,20 +329,20 @@ class TestResponseBase(ABC):
|
|||
r = self.response_class("http://example.com")
|
||||
if self.response_class == Response:
|
||||
with pytest.raises(TypeError):
|
||||
list(r.follow_all(urls=None))
|
||||
list(r.follow_all(urls=None)) # type: ignore[arg-type]
|
||||
with pytest.raises(TypeError):
|
||||
list(r.follow_all(urls=12345))
|
||||
list(r.follow_all(urls=12345)) # type: ignore[arg-type]
|
||||
with pytest.raises(ValueError, match="url can't be None"):
|
||||
list(r.follow_all(urls=[None]))
|
||||
list(r.follow_all(urls=[None])) # type: ignore[list-item]
|
||||
else:
|
||||
with pytest.raises(
|
||||
ValueError, match="Please supply exactly one of the following arguments"
|
||||
):
|
||||
list(r.follow_all(urls=None))
|
||||
list(r.follow_all(urls=None)) # type: ignore[arg-type]
|
||||
with pytest.raises(TypeError):
|
||||
list(r.follow_all(urls=12345))
|
||||
list(r.follow_all(urls=12345)) # type: ignore[arg-type]
|
||||
with pytest.raises(ValueError, match="url can't be None"):
|
||||
list(r.follow_all(urls=[None]))
|
||||
list(r.follow_all(urls=[None])) # type: ignore[list-item]
|
||||
|
||||
@pytest.mark.xfail(
|
||||
not W3LIB_STRIPS_URLS,
|
||||
|
|
@ -384,14 +388,14 @@ class TestResponseBase(ABC):
|
|||
|
||||
def _assert_followed_url(
|
||||
self,
|
||||
follow_obj: str | Link,
|
||||
follow_obj: str | Link | Selector,
|
||||
target_url: str,
|
||||
response: Response | None = None,
|
||||
encoding: str | None = None,
|
||||
) -> None:
|
||||
if response is None:
|
||||
response = self._links_response()
|
||||
req = response.follow(follow_obj)
|
||||
req = response.follow(follow_obj) # type: ignore[arg-type]
|
||||
assert req.url == target_url
|
||||
if encoding is not None:
|
||||
assert req.encoding == encoding
|
||||
|
|
|
|||
7
tox.ini
7
tox.ini
|
|
@ -5,7 +5,8 @@
|
|||
|
||||
[tox]
|
||||
requires =
|
||||
sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8
|
||||
sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10
|
||||
tox-uv
|
||||
envlist =
|
||||
pre-commit
|
||||
pylint
|
||||
|
|
@ -194,8 +195,8 @@ deps =
|
|||
brotlicffi==1.2.0.0; implementation_name == "pypy"
|
||||
google-cloud-storage==1.29.0
|
||||
httpx2[http2,socks]==2.0.0
|
||||
ipython==7.1.0
|
||||
ptpython==2.0.1
|
||||
ipython==8.15.0
|
||||
ptpython==3.0.23
|
||||
robotexclusionrulesparser==1.6.2
|
||||
uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"
|
||||
zstandard==0.16.0; implementation_name != "pypy"
|
||||
|
|
|
|||
Loading…
Reference in New Issue