RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into feed-mode

This commit is contained in:
Adrian Chaves 2026-08-04 11:42:31 +02:00
parent 4b14ff24cb 4cdde1c79c
commit 76d7a1a660
97 changed files with 2638 additions and 837 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/dependabot.yml vendored Normal file
View File

@ -0,0 +1,12 @@
version: 2
updates:
- package-ecosystem: github-actions
directory: "/"
schedule:
interval: monthly
groups:
github-actions:
patterns:
- "*"
cooldown:
default-days: 7

6
.github/workflows/auto-close-llm-pr.yml vendored
View File

@ -1,5 +1,7 @@
name: Auto-close LLM PRs
on:
# The workflow only reads the pull request body through the API, it never
# checks out or runs pull request code, so pull_request_target is safe here.
on: # zizmor: ignore[dangerous-triggers]
pull_request_target:
types: [opened]
permissions:
@ -11,7 +13,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check PR body and close if LLM-written
uses: actions/github-script@v6
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |

32
.github/workflows/checks.yml vendored
View File

@ -1,4 +1,8 @@
name: Checks
permissions:
contents: read
on:
push:
branches:
@ -13,6 +17,10 @@ concurrency:
jobs:
checks:
runs-on: ubuntu-latest
env:
# Make uv use the interpreter that actions/setup-python installed instead
# of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@ -38,21 +46,31 @@ jobs:
TOXENV: twinecheck
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
docs/requirements.txt
pyproject.toml
tox.ini
- name: Run check
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
run: uvx --with tox-uv tox
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: pre-commit/action@v3.0.1
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1

59
.github/workflows/codspeed.yml vendored Normal file
View File

@ -0,0 +1,59 @@
---
name: codspeed
on:
push:
branches:
- master
pull_request:
paths:
- scrapy/**
- tests/benchmarks/**
- .github/workflows/codspeed.yml
- tox.ini
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
permissions: {}
jobs:
benchmark:
runs-on: ubuntu-latest
env:
# Make uv use the interpreter that actions/setup-python installed
# instead of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
permissions:
contents: read
id-token: write # OIDC authentication with CodSpeed
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python 3.14
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.14'
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
pyproject.toml
tox.ini
- name: Install dependencies
# tox must stay on PATH for the CodSpeed action to invoke it.
run: |
uv tool install --with tox-uv tox
tox -n -e benchmark
- name: Run benchmarks
uses: CodSpeedHQ/action@0ca9cbbf4623b599a6c3ed4fc8a922942705d9f1 # v5.0.2
with:
mode: simulation
run: tox -e benchmark

35
.github/workflows/publish.yml vendored
View File

@ -1,4 +1,8 @@
name: Publish
permissions:
contents: read
on:
push:
tags:
@ -9,8 +13,28 @@ concurrency:
cancel-in-progress: true
jobs:
build:
name: Build distribution
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.14"
- run: |
python -m pip install --upgrade build
python -m build
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: python-package-distributions
path: dist/
publish:
name: Upload release to PyPI
needs:
- build
runs-on: ubuntu-latest
environment:
name: pypi
@ -18,12 +42,9 @@ jobs:
permissions:
id-token: write
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
python-version: "3.14"
- run: |
python -m pip install --upgrade build
python -m build
name: python-package-distributions
path: dist/
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2

55
.github/workflows/tests-macos.yml vendored
View File

@ -1,4 +1,8 @@
name: macOS
permissions:
contents: read
on:
push:
branches:
@ -12,39 +16,62 @@ concurrency:
jobs:
tests:
name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})
runs-on: macos-latest
env:
PYTEST_ADDOPTS: -n auto
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
# Make uv use the interpreter that actions/setup-python installed instead
# of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
env:
- TOXENV: py
include:
- python-version: '3.14'
env:
TOXENV: no-reactor
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.14"
env:
TOXENV: py
coverage: true
- python-version: "3.14"
env:
TOXENV: no-reactor
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
pyproject.toml
tox.ini
- name: Install mitmproxy
env:
# mitmproxy needs a newer Python than the oldest matrix entries, so let
# uv download one where no system interpreter is new enough.
UV_PYTHON_PREFERENCE: system
run: uv tool install mitmproxy
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
run: uvx --with tox-uv tox
- name: Upload coverage report
uses: codecov/codecov-action@v5
if: ${{ matrix.coverage }}
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
report_type: test_results

57
.github/workflows/tests-ubuntu.yml vendored
View File

@ -1,4 +1,8 @@
name: Ubuntu
permissions:
contents: read
on:
push:
branches:
@ -12,9 +16,13 @@ concurrency:
jobs:
tests:
name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})
runs-on: ubuntu-latest
env:
PYTEST_ADDOPTS: -n auto
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
# Make uv use the interpreter that actions/setup-python installed instead
# of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@ -34,27 +42,25 @@ jobs:
- python-version: "3.14"
env:
TOXENV: py
coverage: true
- python-version: "3.14"
env:
TOXENV: default-reactor
coverage: true
- python-version: "3.14"
env:
TOXENV: no-reactor
# pinned due to https://github.com/pypy/pypy/issues/5388
- python-version: pypy3.11-7.3.20
env:
TOXENV: pypy3
coverage: true
# min deps
- python-version: "3.10.19"
env:
TOXENV: min
coverage: true
- python-version: "3.10.19"
env:
TOXENV: min-default-reactor
- python-version: "3.10.19"
env:
TOXENV: min-no-reactor
coverage: true
# pinned due to https://github.com/pypy/pypy/issues/5388
- python-version: pypy3.11-7.3.20
env:
@ -62,16 +68,20 @@ jobs:
- python-version: "3.10.19"
env:
TOXENV: min-extra-deps
coverage: true
- python-version: "3.10.19"
env:
TOXENV: min-botocore
coverage: true
- python-version: "3.14"
env:
TOXENV: extra-deps
coverage: true
- python-version: "3.14"
env:
TOXENV: no-reactor-extra-deps
coverage: true
# pinned due to https://github.com/pypy/pypy/issues/5388
- python-version: pypy3.11-7.3.20
env:
@ -79,12 +89,15 @@ jobs:
- python-version: "3.14"
env:
TOXENV: botocore
coverage: true
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
@ -94,20 +107,32 @@ jobs:
sudo apt-get update
sudo apt-get install libxml2-dev libxslt-dev
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
pyproject.toml
tox.ini
- name: Install mitmproxy
run: pipx install mitmproxy
env:
# mitmproxy needs a newer Python than the oldest matrix entries, so let
# uv download one where no system interpreter is new enough.
UV_PYTHON_PREFERENCE: system
# mitmproxy has no PyPy wheels, so run it on CPython regardless of the
# interpreter under test.
run: uv tool install --python cpython mitmproxy
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
run: uvx --with tox-uv tox
- name: Upload coverage report
uses: codecov/codecov-action@v5
if: ${{ matrix.coverage }}
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
report_type: test_results

52
.github/workflows/tests-windows.yml vendored
View File

@ -1,4 +1,8 @@
name: Windows
permissions:
contents: read
on:
push:
branches:
@ -12,9 +16,13 @@ concurrency:
jobs:
tests:
name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})
runs-on: windows-latest
env:
PYTEST_ADDOPTS: -n auto
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
# Make uv use the interpreter that actions/setup-python installed instead
# of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@ -22,21 +30,10 @@ jobs:
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.11"
env:
TOXENV: py
- python-version: "3.12"
env:
TOXENV: py
- python-version: "3.13"
env:
TOXENV: py
- python-version: "3.14"
env:
TOXENV: py
- python-version: "3.14"
env:
TOXENV: default-reactor
coverage: true
- python-version: "3.14"
env:
TOXENV: no-reactor
@ -54,24 +51,39 @@ jobs:
TOXENV: extra-deps
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
pyproject.toml
tox.ini
- name: Install mitmproxy
env:
# mitmproxy needs a newer Python than the oldest matrix entries, so let
# uv download one where no system interpreter is new enough.
UV_PYTHON_PREFERENCE: system
run: uv tool install mitmproxy
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
run: uvx --with tox-uv tox
- name: Upload coverage report
uses: codecov/codecov-action@v5
if: ${{ matrix.coverage }}
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
report_type: test_results

7
.pre-commit-config.yaml
View File

@ -27,6 +27,11 @@ repos:
hooks:
- id: sphinx-lint
- repo: https://github.com/scrapy/sphinx-scrapy
rev: 0.8.8
rev: 0.8.10
hooks:
- id: sphinx-scrapy
- repo: https://github.com/zizmorcore/zizmor-pre-commit
rev: v1.28.0
hooks:
- id: zizmor
args: [--no-progress, --fix]

16
README.rst
View File

@ -5,7 +5,7 @@
:alt: Scrapy
:width: 480px
|version| |python_version| |ubuntu| |macos| |windows| |coverage| |conda| |deepwiki|
|version| |python_version| |tests| |coverage| |conda| |deepwiki|
.. |version| image:: https://img.shields.io/pypi/v/Scrapy.svg
:target: https://pypi.org/pypi/Scrapy
@ -15,17 +15,9 @@
:target: https://pypi.org/pypi/Scrapy
:alt: Supported Python Versions
.. |ubuntu| image:: https://github.com/scrapy/scrapy/workflows/Ubuntu/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AUbuntu
:alt: Ubuntu
.. |macos| image:: https://github.com/scrapy/scrapy/workflows/macOS/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AmacOS
:alt: macOS
.. |windows| image:: https://github.com/scrapy/scrapy/workflows/Windows/badge.svg
:target: https://github.com/scrapy/scrapy/actions?query=workflow%3AWindows
:alt: Windows
.. |tests| image:: https://img.shields.io/github/check-runs/scrapy/scrapy/master?label=tests
:target: https://github.com/scrapy/scrapy/actions?query=branch%3Amaster
:alt: Tests
.. |coverage| image:: https://img.shields.io/codecov/c/github/scrapy/scrapy/master.svg
:target: https://codecov.io/github/scrapy/scrapy?branch=master

3
conftest.py
View File

@ -54,6 +54,9 @@ if not H2_ENABLED:
if find_spec("httpx2") is None and find_spec("httpx") is None:
collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
if find_spec("pytest_codspeed") is None:
collect_ignore.append("tests/benchmarks")
def pytest_addoption(parser, pluginmanager):
if pluginmanager.hasplugin("twisted"):

16
docs/faq.rst
View File

@ -220,21 +220,15 @@ the :ref:`topics-signals-ref` to know which ones.
What does the response status code 999 mean?
--------------------------------------------
999 is a custom response status code used by Yahoo sites to throttle requests.
999 is a custom response status code used by some sites to throttle requests.
Try slowing down the crawling speed by using a download delay of ``2`` (or
higher) in your spider:
higher) for the affected domains, with the :setting:`DOWNLOAD_SLOTS` setting:
.. code-block:: python
from scrapy.spiders import CrawlSpider
class MySpider(CrawlSpider):
name = "myspider"
download_delay = 2
# [ ... rest of the spider code ... ]
DOWNLOAD_SLOTS = {
"example.com": {"delay": 2},
}
Or by setting a global download delay in your project with the
:setting:`DOWNLOAD_DELAY` setting.

3
docs/news.rst
View File

@ -1417,7 +1417,8 @@ Deprecations
- ``download_warnsize`` (use :setting:`DOWNLOAD_WARNSIZE`)
- ``max_concurrent_requests`` (use :setting:`CONCURRENT_REQUESTS`)
- ``max_concurrent_requests`` (use
:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`)
- ``user_agent`` (use :setting:`USER_AGENT`)

2
docs/requirements.in
View File

@ -5,4 +5,4 @@ sphinx
sphinx-notfound-page
sphinx-rtd-theme
sphinx-rtd-dark-mode
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10

2
docs/requirements.txt
View File

@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0
# via
# -r docs/requirements.in
# sphinx-rtd-dark-mode
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@c0b2ac815afc3cb8857d575cecb5d55c05e6b737
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8
# via -r docs/requirements.in
sphinx-sitemap==2.9.0
# via sphinx-scrapy

1
docs/topics/asyncio.rst
View File

@ -267,6 +267,7 @@ Here are some examples of APIs and patterns that need a replacement:
Scrapy provides unified helpers for some of these examples:
.. autofunction:: scrapy.utils.asyncio.sleep
.. autofunction:: scrapy.utils.asyncio.call_later
.. autofunction:: scrapy.utils.asyncio.create_looping_call
.. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall

7
docs/topics/autothrottle.rst
View File

@ -106,10 +106,9 @@ delay of its download slot:
Request("https://example.com", meta={"autothrottle_dont_adjust_delay": True})
Note, however, that AutoThrottle still determines the starting delay of every
download slot by setting the ``download_delay`` attribute on the running
spider. If you want AutoThrottle not to impact a download slot at all, in
addition to setting this meta key in all requests that use that download slot,
you might want to set a custom value for the ``delay`` attribute of that
download slot. If you want AutoThrottle not to impact a download slot at all,
in addition to setting this meta key in all requests that use that download
slot, you might want to set a custom value for the ``delay`` attribute of that
download slot, e.g. using :setting:`DOWNLOAD_SLOTS`.
Settings

13
docs/topics/commands.rst
View File

@ -114,8 +114,8 @@ some usage help and the available commands::
scrapy <command> [options] [args]
Available commands:
crawl Run a spider
fetch Fetch a URL using the Scrapy downloader
runspider Run a spider from a Python file, no project required
[...]
The first line will print the currently active project if you're inside a
@ -263,7 +263,9 @@ crawl
* Syntax: ``scrapy crawl <spider>``
* Requires project: *yes*
Start crawling using a spider.
Start crawling using the spider with the given :attr:`~scrapy.Spider.name`,
which must be one of those that :command:`list` reports. To run a spider from a
file instead, use :command:`runspider`.
Supported options:
@ -571,8 +573,9 @@ runspider
* Syntax: ``scrapy runspider <spider_file.py>``
* Requires project: *no*
Run a spider self-contained in a Python file, without having to create a
project.
Run the spider defined in the given Python file, without requiring a project.
Supported options: the same as :command:`crawl`.
Example usage::
@ -665,6 +668,8 @@ Example:
COMMANDS_MODULE = "mybot.commands"
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
.. _Deploying your project: https://scrapyd.readthedocs.io/en/latest/deploy.html
Register commands via setup.py entry points

47
docs/topics/components.rst
View File

@ -9,37 +9,22 @@ A Scrapy component is any class whose objects are built using
That includes the classes that you may assign to the following settings:
- :setting:`ADDONS`
- :setting:`TWISTED_DNS_RESOLVER`
- :setting:`DOWNLOAD_HANDLERS`
- :setting:`DOWNLOADER_MIDDLEWARES`
- :setting:`DUPEFILTER_CLASS`
- :setting:`EXTENSIONS`
- :setting:`FEED_EXPORTERS`
- :setting:`FEED_STORAGES`
- :setting:`ITEM_PIPELINES`
- :setting:`SCHEDULER`
- :setting:`SCHEDULER_DISK_QUEUE`
- :setting:`SCHEDULER_MEMORY_QUEUE`
- :setting:`SCHEDULER_PRIORITY_QUEUE`
- :setting:`SCHEDULER_START_DISK_QUEUE`
- :setting:`SCHEDULER_START_MEMORY_QUEUE`
- :setting:`SPIDER_MIDDLEWARES`
- :setting:`ADDONS`
- :setting:`DOWNLOAD_HANDLERS`
- :setting:`DOWNLOADER_MIDDLEWARES`
- :setting:`DUPEFILTER_CLASS`
- :setting:`EXTENSIONS`
- :setting:`FEED_EXPORTERS`
- :setting:`FEED_STORAGES`
- :setting:`ITEM_PIPELINES`
- :setting:`SCHEDULER`
- :setting:`SCHEDULER_DISK_QUEUE`
- :setting:`SCHEDULER_MEMORY_QUEUE`
- :setting:`SCHEDULER_PRIORITY_QUEUE`
- :setting:`SCHEDULER_START_DISK_QUEUE`
- :setting:`SCHEDULER_START_MEMORY_QUEUE`
- :setting:`SPIDER_MIDDLEWARES`
- :setting:`TWISTED_DNS_RESOLVER`
Third-party Scrapy components may also let you define additional Scrapy
components, usually configurable through :ref:`settings <topics-settings>`, to

21
docs/topics/extensions.rst
View File

@ -136,18 +136,10 @@ Core Stats extension
Enable the collection of core statistics, provided the stats collection is
enabled (see :ref:`topics-stats`).
The following stats are collected:
* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`).
* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`).
* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`).
* ``finish_reason``: the closing reason string (e.g. ``"finished"``,
``"closespider_timeout"``).
* ``item_scraped_count``: total number of items that passed all pipelines.
* ``item_dropped_count``: total number of items dropped by a pipeline.
* ``item_dropped_reasons_count/<ExceptionName>``: per-exception drop count
(e.g. ``item_dropped_reasons_count/DropItem``).
* ``response_received_count``: total number of HTTP responses received.
The following stats are collected: :stat:`elapsed_time_seconds`,
:stat:`finish_reason`, :stat:`finish_time`, :stat:`item_dropped_count`,
:stat:`item_dropped_reasons_count/{exception}`, :stat:`item_scraped_count`,
:stat:`response_received_count`, :stat:`start_time`.
Log Count extension
~~~~~~~~~~~~~~~~~~~
@ -190,7 +182,7 @@ Monitors the memory used by the Scrapy process that runs the spider and:
1. sends a :signal:`memusage_warning_reached` signal when it exceeds
:setting:`MEMUSAGE_WARNING_MB`
2. closes the spider with the `"memusage_exceeded"` reason when it exceeds
2. closes the spider with the ``"memusage_exceeded"`` reason when it exceeds
:setting:`MEMUSAGE_LIMIT_MB`
This extension is enabled by the :setting:`MEMUSAGE_ENABLED` setting and
@ -214,7 +206,8 @@ An extension for debugging memory usage. It collects information about:
* objects left alive that shouldn't. For more info, see :ref:`topics-leaks-trackrefs`
To enable this extension, turn on the :setting:`MEMDEBUG_ENABLED` setting. The
info will be stored in the stats.
info will be stored in the :stat:`memdebug/gc_garbage_count` and
:stat:`memdebug/live_refs/{cls}` stats.
.. _topics-extensions-ref-spiderstate:

8
docs/topics/jobs.rst
View File

@ -83,6 +83,14 @@ stopping it cleanly. Forced, sudden or otherwise unclean shutdown can lead to
data corruption in the job directory, which may prevent the spider from
resuming correctly.
Scrapy version changes
----------------------
The contents of a job directory are an implementation detail of the Scrapy
version that wrote them. A job must be resumed with the same Scrapy version
that paused it; after upgrading or downgrading Scrapy, start a new job with a
new job directory.
Feed exports
------------

27
docs/topics/settings.rst
View File

@ -305,10 +305,21 @@ These settings cannot be :ref:`set from a spider <spider-settings>`.
These settings are:
- :setting:`TWISTED_REACTOR_ENABLED`
- :setting:`ADDONS`
- :setting:`COMMANDS_MODULE`
- :setting:`FORCE_CRAWLER_PROCESS`
- :setting:`SPIDER_LOADER_CLASS` and settings used by the corresponding
spider loader class, e.g. :setting:`SPIDER_MODULES` and
:setting:`SPIDER_LOADER_WARN_ONLY` for the default spider loader class.
- :setting:`TWISTED_REACTOR_ENABLED`
:setting:`ADDONS` is a special case: it can be set from a spider, but the
``update_pre_crawler_settings()`` method of :ref:`add-ons <topics-addons>`
enabled that way is not called.
:setting:`TWISTED_REACTOR` also acts as a pre-crawler setting when running a
:ref:`command that needs a CrawlerProcess <topics-commands-crawlerprocess>`,
since its project-level value determines the crawler process class.
.. _reactor-settings:
@ -409,6 +420,9 @@ Default: ``{}``
A dict containing paths to the add-ons enabled in your project and their
priorities. For more information, see :ref:`topics-addons`.
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`, with a
caveat described in that section.
.. setting:: ASYNCIO_EVENT_LOOP
ASYNCIO_EVENT_LOOP
@ -561,7 +575,7 @@ CONCURRENT_REQUESTS
Default: ``16``
The maximum number of concurrent (i.e. simultaneous) requests that will be
performed by the Scrapy downloader.
performed by the Scrapy downloader. Use ``0`` for no limit.
.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
@ -939,10 +953,6 @@ desired.
.. _spider-download_delay-attribute:
.. note::
This delay can be set per spider using :attr:`download_delay` spider attribute.
It is possible to change this setting per domain by using
:setting:`DOWNLOAD_SLOTS`.
@ -1402,6 +1412,8 @@ When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``,
Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a
non-default value in :ref:`per-spider settings <spider-settings>`.
.. note:: This is a :ref:`pre-crawler setting <pre-crawler-settings>`.
.. setting:: FTP_PASSIVE_MODE
FTP_PASSIVE_MODE
@ -1855,7 +1867,8 @@ Default: ``False``
Setting to ``True`` will log debug information about the requests scheduler.
This currently logs (only once) if the requests cannot be serialized to disk.
Stats counter (``scheduler/unserializable``) tracks the number of times this happens.
The :stat:`scheduler/unserializable` stat tracks the number of times this
happens.
Example entry in logs::

21
docs/topics/signals.rst
View File

@ -504,6 +504,27 @@ headers_received
:param spider: the spider associated with the response
:type spider: :class:`~scrapy.Spider` object
robots_parsed
~~~~~~~~~~~~~
.. signal:: robots_parsed
.. function:: robots_parsed(robotparser, request)
.. versionadded:: VERSION
Sent by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it
downloads and parses a :file:`robots.txt` file, for the host that *request*
targets.
This signal supports :ref:`asynchronous handlers <signal-deferred>`.
:param robotparser: the parser holding the parsed :file:`robots.txt` contents
:type robotparser: :class:`~scrapy.robotstxt.RobotParser` object
:param request: the request that triggered the :file:`robots.txt` download
:type request: :class:`~scrapy.Request` object
Response signals
----------------

641
docs/topics/stats.rst
View File

@ -21,6 +21,8 @@ using the Stats Collector from.
Another feature of the Stats Collector is that it's very efficient (when
enabled) and extremely efficient (almost unnoticeable) when disabled.
See :ref:`topics-stats-reference` below for the stats that Scrapy sets.
.. _topics-stats-usecases:
Common Stats Collector uses
@ -101,3 +103,642 @@ DummyStatsCollector
-------------------
.. autoclass:: DummyStatsCollector
.. _topics-stats-reference:
Built-in stats reference
========================
Scrapy sets the following :ref:`stats <topics-stats>`. Components other than
those built into Scrapy may set additional stats; see their documentation.
Stat keys that contain a ``{placeholder}`` below stand for a family of stats,
one per actual value of the placeholder.
.. note:: Most stats are set by a specific :ref:`component
<topics-components>`, and are only present if that component is enabled and
its code path is reached. A stat that is missing from
:meth:`~scrapy.statscollectors.StatsCollector.get_stats` output is
equivalent to a counter of 0.
.. stat:: downloader/exception_count
``downloader/exception_count``
Number of exceptions raised while downloading requests.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/exception_type_count/{exception_type}
``downloader/exception_type_count/{exception_type}``
Number of exceptions raised while downloading requests, per exception type,
where ``{exception_type}`` is the import path of the exception class, e.g.
``twisted.internet.error.DNSLookupError``.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/request_bytes
``downloader/request_bytes``
Total size, in bytes, of the requests sent, counting the request line, the
headers and the body. As with :stat:`downloader/request_count`, requests
served from the cache are also counted.
It is an approximation, reconstructed from each :class:`~scrapy.Request`
object instead of measured on the wire, so it does not account for the
actual bytes that the :ref:`download handler
<topics-download-handlers>` sends, e.g. transport-level overhead.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/request_count
``downloader/request_count``
Number of requests sent.
Requests that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
serves from the cache are also counted, even though they are never sent,
because it handles requests after
:class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/request_method_count/{method}
``downloader/request_method_count/{method}``
Number of requests sent, per HTTP method, e.g. ``GET`` or ``POST``. As with
:stat:`downloader/request_count`, requests served from the cache are also
counted.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/response_bytes
``downloader/response_bytes``
Total size, in bytes, of the responses received, counting the status line,
the headers and the body. It covers the same responses as
:stat:`downloader/response_count`.
The body is counted as received, i.e. still compressed for responses that
used ``Content-Encoding``, because
:class:`~scrapy.downloadermiddlewares.stats.DownloaderStats` handles
responses before
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
decompresses them. See :stat:`httpcompression/response_bytes` for
decompressed sizes.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/response_count
``downloader/response_count``
Number of responses received.
It counts responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
serves from the cache, even though they do not come from the network, and
responses that a downloader middleware consumes before they reach your
spider, e.g. redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware`
turns into new requests. Compare with :stat:`response_received_count`.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: downloader/response_status_count/{status_code}
``downloader/response_status_count/{status_code}``
Number of responses received, per HTTP status code, e.g. ``200`` or
``404``. It covers the same responses as :stat:`downloader/response_count`.
Set by :class:`~scrapy.downloadermiddlewares.stats.DownloaderStats`.
.. stat:: dupefilter/filtered
``dupefilter/filtered``
Number of requests dropped as duplicates.
Set by :class:`~scrapy.dupefilters.RFPDupeFilter`.
.. stat:: elapsed_time_seconds
``elapsed_time_seconds``
Time, as a :class:`float`, in seconds, between the :signal:`spider_opened`
and the :signal:`spider_closed` signals.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: feedexport/failed_count/{storage}
``feedexport/failed_count/{storage}``
Number of :ref:`feeds <topics-feed-exports>` that could not be stored, per
:ref:`storage backend <topics-feed-storage-backends>`, where ``{storage}``
is the class name of the storage backend, e.g. ``FileFeedStorage``.
.. stat:: feedexport/success_count/{storage}
``feedexport/success_count/{storage}``
Number of :ref:`feeds <topics-feed-exports>` stored successfully, per
:ref:`storage backend <topics-feed-storage-backends>`, where ``{storage}``
is the class name of the storage backend, e.g. ``FileFeedStorage``.
.. stat:: file_count
``file_count``
Number of files handled by the :ref:`media pipelines
<topics-media-pipeline>`.
.. stat:: file_status_count/{status}
``file_status_count/{status}``
Number of files handled by the :ref:`media pipelines
<topics-media-pipeline>`, per status, where ``{status}`` is one of:
- ``downloaded``: the file was downloaded.
- ``cached``: the file came from the
:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
cache.
- ``uptodate``: the file was already in the storage backend and had not
:ref:`expired <file-expiration>`, so it was not downloaded again.
.. stat:: finish_reason
``finish_reason``
String indicating why the crawl finished. It matches the *reason* argument
of the :signal:`spider_closed` signal.
Scrapy uses the following reasons:
- ``cancelled``: the spider was closed without a more specific reason,
e.g. because :exc:`~scrapy.exceptions.CloseSpider` was raised without
one.
- ``closespider_errorcount``: see :setting:`CLOSESPIDER_ERRORCOUNT`.
- ``closespider_itemcount``: see :setting:`CLOSESPIDER_ITEMCOUNT`.
- ``closespider_pagecount``: see :setting:`CLOSESPIDER_PAGECOUNT`.
- ``closespider_pagecount_no_item``: see
:setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM`.
- ``closespider_timeout``: see :setting:`CLOSESPIDER_TIMEOUT`.
- ``closespider_timeout_no_item``: see
:setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`.
- ``finished``: the spider became idle with no pending requests, i.e. it
finished normally.
- ``memusage_exceeded``: see :setting:`MEMUSAGE_LIMIT_MB`.
- ``shutdown``: the crawl was interrupted, e.g. by a system signal such
as ``SIGINT`` (:kbd:`Ctrl-C`).
Third-party components and your own code may use any other reason, e.g. by
raising :exc:`~scrapy.exceptions.CloseSpider` with it.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: finish_time
``finish_time``
Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when
the :signal:`spider_closed` signal was sent.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: httpcache/errorrecovery
``httpcache/errorrecovery``
Number of times that a stale cached response was used because downloading a
fresh response raised an exception.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/firsthand
``httpcache/firsthand``
Number of responses that were downloaded without a matching cache entry to
validate against, i.e. responses for requests counted in
:stat:`httpcache/miss`.
It is lower than :stat:`httpcache/miss` when some of those requests yield
no response, either because they are dropped (see
:stat:`httpcache/ignore`) or because their download fails.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/hit
``httpcache/hit``
Number of requests served from the cache.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/ignore
``httpcache/ignore``
Number of requests dropped because they were not in the cache and
:setting:`HTTPCACHE_IGNORE_MISSING` is ``True``.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/invalidate
``httpcache/invalidate``
Number of times that a cached response failed validation and was replaced
with a freshly downloaded response.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/miss
``httpcache/miss``
Number of requests for which no cache entry could be read, either because
there was none or because reading it failed, in which case the request is
also counted in :stat:`httpcache/retrieve_error`. Those requests are
downloaded (see :stat:`httpcache/firsthand`), or dropped if
:setting:`HTTPCACHE_IGNORE_MISSING` is ``True`` (see
:stat:`httpcache/ignore`).
Requests with a stale cache entry are not counted here; see
:stat:`httpcache/revalidate` and :stat:`httpcache/invalidate`.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/retrieve_error
``httpcache/retrieve_error``
Number of cache entries that could not be read, and hence were treated as
cache misses. Those requests are also counted in :stat:`httpcache/miss`.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/revalidate
``httpcache/revalidate``
Number of times that a cached response was successfully validated against
the target server, and hence used instead of the fresh response.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/store
``httpcache/store``
Number of responses stored in the cache.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcache/uncacheable
``httpcache/uncacheable``
Number of responses not stored in the cache because the
:setting:`HTTPCACHE_POLICY` did not allow it.
Every response considered for caching is counted either here or in
:stat:`httpcache/store`, so ``httpcache/store + httpcache/uncacheable``
equals ``httpcache/firsthand + httpcache/invalidate``.
Set by :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`.
.. stat:: httpcompression/response_bytes
``httpcompression/response_bytes``
Total size, in bytes, of decompressed response bodies, counting only the
body and only responses that were actually decompressed. Compare with
:stat:`downloader/response_bytes`.
Set by
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`.
.. stat:: httpcompression/response_count
``httpcompression/response_count``
Number of decompressed responses.
Set by
:class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`.
.. stat:: httperror/response_ignored_count
``httperror/response_ignored_count``
Number of responses dropped because of their HTTP status code.
Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`.
.. stat:: httperror/response_ignored_status_count/{status_code}
``httperror/response_ignored_status_count/{status_code}``
Number of responses dropped because of their HTTP status code, per HTTP
status code, e.g. ``404``.
Set by :class:`~scrapy.spidermiddlewares.httperror.HttpErrorMiddleware`.
.. stat:: item_dropped_count
``item_dropped_count``
Number of items dropped by an :ref:`item pipeline
<topics-item-pipeline>`, i.e. number of times that the
:signal:`item_dropped` signal was sent.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: item_dropped_reasons_count/{exception}
``item_dropped_reasons_count/{exception}``
Number of items dropped, per exception, where ``{exception}`` is the class
name of the exception that caused the item to be dropped.
Only :exc:`~scrapy.exceptions.DropItem` and its subclasses drop items, and
each one is counted under its own class name, e.g.
``item_dropped_reasons_count/DropItem`` for
:exc:`~scrapy.exceptions.DropItem` itself and
``item_dropped_reasons_count/MyDropItem`` for a ``MyDropItem`` subclass of
it. Any other exception raised by an :ref:`item pipeline
<topics-item-pipeline>` triggers the :signal:`item_error` signal instead of
:signal:`item_dropped`, and is not counted here or in
:stat:`item_dropped_count`.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: item_scraped_count
``item_scraped_count``
Number of items that passed all :ref:`item pipelines
<topics-item-pipeline>`, i.e. number of times that the
:signal:`item_scraped` signal was sent.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: items_per_minute
``items_per_minute``
Average number of items scraped per minute during the crawl.
It is ``None`` if the crawl took less than a minute.
Set by :class:`~scrapy.extensions.logstats.LogStats`.
.. stat:: log_count/{level}
``log_count/{level}``
Number of log messages, per logging level name, e.g. ``INFO`` or
``WARNING``.
Only messages that the :setting:`LOG_LEVEL` setting allows are counted.
Set by :class:`~scrapy.extensions.logcount.LogCount`.
.. stat:: memdebug/gc_garbage_count
``memdebug/gc_garbage_count``
Number of objects in :data:`gc.garbage` when the spider is closed.
Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires
:setting:`MEMDEBUG_ENABLED` to be ``True``.
.. stat:: memdebug/live_refs/{cls}
``memdebug/live_refs/{cls}``
Number of live objects of class ``{cls}`` when the spider is closed, as
reported by :ref:`trackref <topics-leaks-trackrefs>`, e.g.
``memdebug/live_refs/HtmlResponse``.
Only set for classes with at least 1 live object.
Set by :class:`~scrapy.extensions.memdebug.MemoryDebugger`, which requires
:setting:`MEMDEBUG_ENABLED` to be ``True``.
.. stat:: memusage/limit_reached
``memusage/limit_reached``
``1`` if memory usage exceeded :setting:`MEMUSAGE_LIMIT_MB`, which also
stops the crawl.
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
.. stat:: memusage/max
``memusage/max``
Maximum peak memory usage, in bytes, observed during the crawl.
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
.. stat:: memusage/startup
``memusage/startup``
Peak memory usage, in bytes, when the engine started.
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
.. stat:: memusage/warning_reached
``memusage/warning_reached``
``1`` if memory usage exceeded :setting:`MEMUSAGE_WARNING_MB`.
Set by :class:`~scrapy.extensions.memusage.MemoryUsage`.
.. stat:: offsite/domains
``offsite/domains``
Number of distinct domains for which at least 1 request was dropped for
being offsite.
Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`.
.. stat:: offsite/filtered
``offsite/filtered``
Number of requests dropped for being offsite.
Set by :class:`~scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`.
.. stat:: request_depth_count/{depth}
``request_depth_count/{depth}``
Number of requests scheduled at depth ``{depth}``, e.g.
``request_depth_count/2``.
Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`, which
requires :setting:`DEPTH_STATS_VERBOSE` to be ``True`` for this stat.
.. stat:: request_depth_max
``request_depth_max``
Maximum depth reached.
Set by :class:`~scrapy.spidermiddlewares.depth.DepthMiddleware`.
.. stat:: response_received_count
``response_received_count``
Number of responses received, i.e. number of times that the
:signal:`response_received` signal was sent.
Unlike :stat:`downloader/response_count`, it does not count responses that
a downloader middleware consumes before they reach the engine, e.g.
redirect responses that :class:`~scrapy.downloadermiddlewares.redirect.RedirectMiddleware`
turns into new requests. Both count responses that :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`
serves from the cache.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: responses_per_minute
``responses_per_minute``
Average number of responses received per minute during the crawl.
It is ``None`` if the crawl took less than a minute.
Set by :class:`~scrapy.extensions.logstats.LogStats`.
.. stat:: retry/count
``retry/count``
Number of requests retried.
Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses.
.. stat:: retry/max_reached
``retry/max_reached``
Number of requests that were not retried because they had already been
retried :setting:`RETRY_TIMES` times.
Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses.
.. stat:: retry/reason_count/{reason}
``retry/reason_count/{reason}``
Number of requests retried, per reason, e.g.
``retry/reason_count/twisted.internet.error.TimeoutError`` or
``retry/reason_count/504 Gateway Time-out``.
Set by :func:`~scrapy.downloadermiddlewares.retry.get_retry_request`, which
:class:`~scrapy.downloadermiddlewares.retry.RetryMiddleware` uses.
.. note:: Code calling
:func:`~scrapy.downloadermiddlewares.retry.get_retry_request` may pass a
custom *stats_base_key*, in which case ``retry`` is replaced with that key
in the 3 stats above.
.. stat:: robotstxt/exception_count/{exception_type}
``robotstxt/exception_count/{exception_type}``
Number of exceptions raised while downloading ``robots.txt`` files, per
exception type, where ``{exception_type}`` is the string representation of
the exception class, e.g. ``<class
'twisted.internet.error.DNSLookupError'>``.
Set by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
.. stat:: robotstxt/forbidden
``robotstxt/forbidden``
Number of requests dropped for being disallowed by ``robots.txt``.
Set by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
.. stat:: robotstxt/request_count
``robotstxt/request_count``
Number of ``robots.txt`` files requested, i.e. 1 per network location for
which at least 1 request was sent.
Set by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
.. stat:: robotstxt/response_count
``robotstxt/response_count``
Number of ``robots.txt`` responses received.
Set by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
.. stat:: robotstxt/response_status_count/{status_code}
``robotstxt/response_status_count/{status_code}``
Number of ``robots.txt`` responses received, per HTTP status code, e.g.
``404``.
Set by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware`.
.. stat:: scheduler/dequeued
``scheduler/dequeued``
Number of requests read from the :ref:`scheduler <topics-scheduler>`.
.. stat:: scheduler/dequeued/disk
``scheduler/dequeued/disk``
Number of requests read from the disk queue of the :ref:`scheduler
<topics-scheduler>`.
.. stat:: scheduler/dequeued/memory
``scheduler/dequeued/memory``
Number of requests read from the memory queue of the :ref:`scheduler
<topics-scheduler>`.
.. stat:: scheduler/enqueued
``scheduler/enqueued``
Number of requests stored into the :ref:`scheduler <topics-scheduler>`.
.. stat:: scheduler/enqueued/disk
``scheduler/enqueued/disk``
Number of requests stored into the disk queue of the :ref:`scheduler
<topics-scheduler>`.
.. stat:: scheduler/enqueued/memory
``scheduler/enqueued/memory``
Number of requests stored into the memory queue of the :ref:`scheduler
<topics-scheduler>`.
.. stat:: scheduler/unserializable
``scheduler/unserializable``
Number of requests that could not be stored into the disk queue of the
:ref:`scheduler <topics-scheduler>` because they could not be
:ref:`serialized <request-serialization>`, and hence were stored into the
memory queue instead.
.. stat:: spider_exceptions/count
``spider_exceptions/count``
Number of unhandled exceptions raised by spider callbacks.
Set by the :ref:`scraper <topics-architecture>`.
.. stat:: spider_exceptions/{exception}
``spider_exceptions/{exception}``
Number of unhandled exceptions raised by spider callbacks, per exception,
where ``{exception}`` is the class name of the exception, e.g.
``spider_exceptions/ValueError``.
Set by the :ref:`scraper <topics-architecture>`.
.. stat:: start_time
``start_time``
Timezone-aware :class:`~datetime.datetime` object, in UTC, indicating when
the :signal:`spider_opened` signal was sent.
Set by :class:`~scrapy.extensions.corestats.CoreStats`.
.. stat:: urllength/request_ignored_count
``urllength/request_ignored_count``
Number of requests dropped for having a URL longer than
:setting:`URLLENGTH_LIMIT`.
Set by :class:`~scrapy.spidermiddlewares.urllength.UrlLengthMiddleware`.

19
extras/qpsclient.py
View File

@ -16,23 +16,20 @@ class QPSSpider(Spider):
name = "qps"
benchurl = "http://localhost:8880/"
# Max concurrency is limited by global CONCURRENT_REQUESTS setting
max_concurrent_requests = 8
# Requests per second goal
qps = None # same as: 1 / download_delay
download_delay = None
qps = None # same as: 1 / DOWNLOAD_DELAY
# time in seconds to delay server responses
latency = None
# number of slots to create
slots = 1
def __init__(self, *a, **kw):
super().__init__(*a, **kw)
if self.qps is not None:
self.qps = float(self.qps)
self.download_delay = 1 / self.qps
elif self.download_delay is not None:
self.download_delay = float(self.download_delay)
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = super().from_crawler(crawler, *args, **kwargs)
if spider.qps is not None:
spider.qps = float(spider.qps)
crawler.settings.set("DOWNLOAD_DELAY", 1 / spider.qps, priority="spider")
return spider
async def start(self):
url = self.benchurl

31
pyproject.toml
View File

@ -69,8 +69,8 @@ brotli = [
gcs = ["google-cloud-storage>=1.29.0"]
httpx = ["httpx2[http2,socks]>=2.0.0"]
images = ["Pillow>=8.3.2"]
ipython = ["ipython>=7.1.0"]
ptpython = ["ptpython>=2.0.1"]
ipython = ["ipython>=8.15.0"]
ptpython = ["ptpython>=3.0.23"]
robotparser = ["robotexclusionrulesparser>=1.6.2"]
s3 = ["boto3>=1.20.0"]
twisted-http2 = ["Twisted[http2]>=21.7.0"]
@ -118,24 +118,10 @@ allow_incomplete_defs = true # 59 errors
# TODO
[[tool.mypy.overrides]]
module = [
"tests.mockserver.*",
"tests.spiders",
"tests.test_closespider",
"tests.test_cmdline",
"tests.test_contracts",
"tests.test_core_downloader",
"tests.test_downloader_handler_twisted_ftp",
"tests.test_downloadermiddleware_cookies",
"tests.test_downloadermiddleware_httpauth",
"tests.test_downloadermiddleware_httpcache",
"tests.test_downloadermiddleware_httpcompression",
"tests.test_downloadermiddleware_httpproxy",
"tests.test_downloadermiddleware_offsite",
"tests.test_downloadermiddleware_redirect",
"tests.test_downloadermiddleware_redirect_base",
"tests.test_downloadermiddleware_redirect_metarefresh",
"tests.test_downloadermiddleware_retry",
"tests.test_downloadermiddleware_robotstxt",
"tests.test_downloaderslotssettings",
"tests.test_dupefilters",
"tests.test_engine_loop",
@ -146,12 +132,6 @@ module = [
"tests.test_feedexport_postprocess",
"tests.test_feedexport_storages",
"tests.test_feedexport_uri_params",
"tests.test_http2_client_protocol",
"tests.test_http_headers",
"tests.test_http_request",
"tests.test_http_request_form",
"tests.test_http_response",
"tests.test_http_response_text",
"tests.test_item",
"tests.test_linkextractors",
"tests.test_loader",
@ -163,11 +143,6 @@ module = [
"tests.test_pipeline_media",
"tests.test_pipelines",
"tests.test_pqueues",
"tests.test_request_attribute_binding",
"tests.test_request_cb_kwargs",
"tests.test_request_dict",
"tests.test_request_left",
"tests.test_robotstxt_interface",
"tests.test_scheduler_base",
"tests.test_settings",
"tests.test_spider",
@ -178,8 +153,6 @@ module = [
"tests.test_squeues",
"tests.test_squeues_request",
"tests.test_stats",
"tests.utils.bases.http_request",
"tests.utils.bases.http_response",
"tests.utils.bases.spider",
]
check_untyped_defs = false

2
scrapy/commands/crawl.py
View File

@ -16,7 +16,7 @@ class Command(BaseRunSpiderCommand):
return "[options] <spider>"
def short_desc(self) -> str:
return "Run a spider"
return "Run a spider of the current project, by name"
def run(self, args: list[str], opts: argparse.Namespace) -> None:
if len(args) < 1:

5
scrapy/commands/genspider.py
View File

@ -32,10 +32,7 @@ def sanitize_module_name(module_name: str) -> str:
def extract_domain(url: str) -> str:
"""Extract domain name from URL string"""
o = urlparse(url)
if o.scheme == "" and o.netloc == "":
o = urlparse("//" + url.lstrip("/"))
return o.netloc
return urlparse(url).netloc
def verify_url_scheme(url: str) -> str:

2
scrapy/commands/parse.py
View File

@ -41,7 +41,7 @@ class Command(BaseRunSpiderCommand):
spider: Spider | None = None
items: ClassVar[dict[int, list[Any]]] = {}
requests: ClassVar[dict[int, list[Request]]] = {}
spidercls: type[Spider] | None
spidercls: type[Spider] | None = None
first_response = None

2
scrapy/commands/runspider.py
View File

@ -38,7 +38,7 @@ class Command(BaseRunSpiderCommand):
return "[options] <spider_file>"
def short_desc(self) -> str:
return "Run a self-contained spider (without creating a project)"
return "Run a spider from a Python file, no project required"
def long_desc(self) -> str:
return "Run the spider defined in the given file"

34
scrapy/core/downloader/__init__.py
View File

@ -27,7 +27,6 @@ from scrapy.utils.defer import (
deferred_from_coro,
maybe_deferred_to_future,
)
from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute
from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
@ -80,22 +79,6 @@ class Slot:
)
def _get_concurrency_delay(
concurrency: int, spider: Spider, settings: BaseSettings
) -> tuple[int, float]:
delay: float = settings.getfloat("DOWNLOAD_DELAY")
if hasattr(spider, "download_delay"):
delay = spider.download_delay
if hasattr(spider, "max_concurrent_requests"): # pragma: no cover
warn_on_deprecated_spider_attribute(
"max_concurrent_requests", "CONCURRENT_REQUESTS"
)
concurrency = spider.max_concurrent_requests
return concurrency, delay
class Downloader:
DOWNLOAD_SLOT = "download_slot"
_SLOT_GC_INTERVAL: float = 60.0 # seconds
@ -112,6 +95,9 @@ class Downloader:
"CONCURRENT_REQUESTS_PER_DOMAIN"
)
self.ip_concurrency: int = self.settings.getint("CONCURRENT_REQUESTS_PER_IP")
# Default delay of new slots. AutoThrottle overrides it to apply
# AUTOTHROTTLE_START_DELAY.
self._delay: float = self.settings.getfloat("DOWNLOAD_DELAY")
self.randomize_delay: bool = self.settings.getbool("RANDOMIZE_DOWNLOAD_DELAY")
self.middleware: DownloaderMiddlewareManager = (
DownloaderMiddlewareManager.from_crawler(crawler)
@ -138,7 +124,8 @@ class Downloader:
self.active.remove(request)
def needs_backout(self) -> bool:
return len(self.active) >= self.total_concurrency
# A total concurrency of 0 means no limit.
return 0 < self.total_concurrency <= len(self.active)
@_warn_spider_arg
def _get_slot(
@ -146,16 +133,11 @@ class Downloader:
) -> tuple[str, Slot]:
key = self.get_slot_key(request)
if key not in self.slots:
assert self.crawler.spider
slot_settings = self.per_slot_settings.get(key, {})
conc = self.ip_concurrency or self.domain_concurrency
conc, delay = _get_concurrency_delay(
conc, self.crawler.spider, self.settings
)
conc, delay = (
slot_settings.get("concurrency", conc),
slot_settings.get("delay", delay),
conc = slot_settings.get(
"concurrency", self.ip_concurrency or self.domain_concurrency
)
delay = slot_settings.get("delay", self._delay)
randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
new_slot = Slot(conc, delay, randomize_delay)
self.slots[key] = new_slot

7
scrapy/core/downloader/handlers/_httpx.py
View File

@ -92,10 +92,11 @@ class HttpxDownloadHandler(_Base):
self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
self._bind_host: str | None = self._get_bind_address_host()
self._limits: httpx.Limits = httpx.Limits(
# hard limit on simultaneous connections
max_connections=self._pool_size_total,
# hard limit on simultaneous connections (None for no limit, which
# is what a CONCURRENT_REQUESTS of 0 means)
max_connections=self._pool_size_total or None,
# total number of idle connections in the pool (extra ones are closed)
max_keepalive_connections=self._pool_size_total,
max_keepalive_connections=self._pool_size_total or None,
)
self._default_client: httpx.AsyncClient = self._make_client()

8
scrapy/core/scheduler.py
View File

@ -366,8 +366,8 @@ class Scheduler(BaseScheduler):
Unless the received request is filtered out by the Dupefilter, attempt to push
it into the disk queue, falling back to pushing it into the memory queue.
Increment the appropriate stats, such as: ``scheduler/enqueued``,
``scheduler/enqueued/disk``, ``scheduler/enqueued/memory``.
Increment the appropriate stats, such as: :stat:`scheduler/enqueued`,
:stat:`scheduler/enqueued/disk`, :stat:`scheduler/enqueued/memory`.
Return ``True`` if the request was stored successfully, ``False`` otherwise.
"""
@ -390,8 +390,8 @@ class Scheduler(BaseScheduler):
falling back to the disk queue if the memory queue is empty.
Return ``None`` if there are no more enqueued requests.
Increment the appropriate stats, such as: ``scheduler/dequeued``,
``scheduler/dequeued/disk``, ``scheduler/dequeued/memory``.
Increment the appropriate stats, such as: :stat:`scheduler/dequeued`,
:stat:`scheduler/dequeued/disk`, :stat:`scheduler/dequeued/memory`.
"""
request: Request | None = self.mqs.pop()
assert self.stats is not None

28
scrapy/crawler.py
View File

@ -100,6 +100,10 @@ class Crawler:
return
self.addons.load_settings(self.settings)
self._apply_deprecated_spider_attr("download_delay", "DOWNLOAD_DELAY")
self._apply_deprecated_spider_attr(
"max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"
)
self.stats = load_object(self.settings["STATS_CLASS"])(self)
lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
@ -155,6 +159,30 @@ class Crawler:
"Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
)
def _apply_deprecated_spider_attr(self, attr: str, setting: str) -> None:
"""Bridge a deprecated spider attribute onto *setting*, warning about
the deprecation (and about being ignored when *setting* is already set
at spider or higher priority)."""
spider = self.spider if self.spider is not None else self.spidercls
if not hasattr(spider, attr):
return
if (self.settings.getpriority(setting) or 0) >= SETTINGS_PRIORITIES["spider"]:
warnings.warn(
f"The {attr!r} spider attribute is deprecated. It is also being "
f"ignored because {setting} is already set at spider or higher "
f"priority. Remove the {attr!r} attribute from your spider.",
category=ScrapyDeprecationWarning,
stacklevel=3,
)
return
warnings.warn(
f"The {attr!r} spider attribute is deprecated. Use the {setting} "
f"setting instead.",
category=ScrapyDeprecationWarning,
stacklevel=3,
)
self.settings.set(setting, getattr(spider, attr), priority="spider")
def _apply_reactorless_default_settings(self) -> None:
"""Change some setting defaults when not using a Twisted reactor.

7
scrapy/downloadermiddlewares/retry.py
View File

@ -14,7 +14,7 @@ from typing import TYPE_CHECKING
from scrapy.exceptions import NotConfigured
from scrapy.utils.decorators import _warn_spider_arg
from scrapy.utils.misc import load_object
from scrapy.utils.misc import _load_objects
from scrapy.utils.python import global_object_name
from scrapy.utils.response import response_status_message
@ -149,10 +149,7 @@ class RetryMiddleware:
self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}
self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
self.give_up_log_level = settings["RETRY_GIVE_UP_LOG_LEVEL"]
self.exceptions_to_retry = tuple(
load_object(x) if isinstance(x, str) else x
for x in settings.getlist("RETRY_EXCEPTIONS")
)
self.exceptions_to_retry = _load_objects(settings.getlist("RETRY_EXCEPTIONS"))
@classmethod
def from_crawler(cls, crawler: Crawler) -> Self:

12
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
from twisted.internet.defer import Deferred
from scrapy import signals
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response
from scrapy.http.request import NO_CALLBACK
@ -98,7 +99,7 @@ class RobotsTxtMiddleware:
assert self.crawler.stats
try:
resp = await self.crawler.engine.download_async(robotsreq)
self._parse_robots(resp, netloc)
await self._parse_robots(resp, netloc, request)
except Exception as e:
if not isinstance(e, IgnoreRequest):
logger.error(
@ -115,13 +116,20 @@ class RobotsTxtMiddleware:
return await maybe_deferred_to_future(parser)
return parser
def _parse_robots(self, response: Response, netloc: str) -> None:
async def _parse_robots(
self, response: Response, netloc: str, request: Request
) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("robotstxt/response_count")
self.crawler.stats.inc_value(
f"robotstxt/response_status_count/{response.status}"
)
rp = self._parserimpl.from_crawler(self.crawler, response.body)
await self.crawler.signals.send_catch_log_async(
signal=signals.robots_parsed,
robotparser=rp,
request=request,
)
rp_dfd = self._parsers[netloc]
assert isinstance(rp_dfd, Deferred)
self._parsers[netloc] = rp

2
scrapy/extensions/logcount.py
View File

@ -20,7 +20,7 @@ class LogCount:
"""Install a log handler that counts log messages by level.
The handler installed is :class:`scrapy.utils.log.LogCounterHandler`.
The counts are stored in stats as ``log_count/<level>``.
The counts are stored in the :stat:`log_count/{level}` stat.
.. versionadded:: 2.14
"""

16
scrapy/extensions/throttle.py
View File

@ -43,18 +43,18 @@ class AutoThrottle:
return cls(crawler)
def _spider_opened(self, spider: Spider) -> None:
self.mindelay = self._min_delay(spider)
self.maxdelay = self._max_delay(spider)
spider.download_delay = self._start_delay(spider) # type: ignore[attr-defined]
self.mindelay = self._min_delay()
self.maxdelay = self._max_delay()
assert self.crawler.engine
self.crawler.engine.downloader._delay = self._start_delay()
def _min_delay(self, spider: Spider) -> float:
s = self.crawler.settings
return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY"))
def _min_delay(self) -> float:
return self.crawler.settings.getfloat("DOWNLOAD_DELAY")
def _max_delay(self, spider: Spider) -> float:
def _max_delay(self) -> float:
return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY")
def _start_delay(self, spider: Spider) -> float:
def _start_delay(self) -> float:
return max(
self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY")
)

4
scrapy/http/request/__init__.py
View File

@ -50,7 +50,9 @@ class VerboseCookie(TypedDict):
secure: NotRequired[bool]
CookiesT: TypeAlias = dict[str | bytes, str | bytes] | list[VerboseCookie]
CookiesT: TypeAlias = (
dict[str | bytes, str | bytes | bool | float | int] | list[VerboseCookie]
)
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")

8
scrapy/http/request/form.py
View File

@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst
from __future__ import annotations
from collections.abc import Iterable
from collections.abc import Iterable, Mapping
from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
from warnings import warn
@ -34,7 +34,7 @@ if TYPE_CHECKING:
FormdataVType: TypeAlias = str | Iterable[str]
FormdataKVType: TypeAlias = tuple[str, FormdataVType]
FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None
FormdataType: TypeAlias = Mapping[str, FormdataVType] | Iterable[FormdataKVType] | None
class FormRequest(Request):
@ -100,7 +100,7 @@ class FormRequest(Request):
super().__init__(*args, **kwargs)
if formdata:
items = formdata.items() if isinstance(formdata, dict) else formdata
items = formdata.items() if isinstance(formdata, Mapping) else formdata
form_query_str = _urlencode(items, self.encoding)
if self.method == "POST":
self.headers.setdefault(
@ -248,7 +248,7 @@ def _get_inputs(
if clickable and clickable[0] not in formdata and clickable[0] is not None:
values.append(clickable)
formdata_items = formdata.items() if isinstance(formdata, dict) else formdata
formdata_items = formdata.items() if isinstance(formdata, Mapping) else formdata
values.extend((k, v) for k, v in formdata_items if v is not None)
return values

21
scrapy/robotstxt.py
View File

@ -67,6 +67,15 @@ class RobotParser(metaclass=ABCMeta):
:type user_agent: str or bytes
"""
def crawl_delay(self, user_agent: str | bytes) -> float | None:
"""Return the ``Crawl-delay`` directive for ``user_agent`` as a number
of seconds, or ``None`` if it is not set or the backend does not support
it.
.. versionadded:: VERSION
"""
return None
class PythonRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@ -85,6 +94,10 @@ class PythonRobotParser(RobotParser):
url = to_unicode(url)
return self.rp.can_fetch(user_agent, url)
def crawl_delay(self, user_agent: str | bytes) -> float | None:
delay = self.rp.crawl_delay(to_unicode(user_agent))
return None if delay is None else float(delay)
class RerpRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@ -105,6 +118,10 @@ class RerpRobotParser(RobotParser):
url = to_unicode(url)
return cast("bool", self.rp.is_allowed(user_agent, url))
def crawl_delay(self, user_agent: str | bytes) -> float | None:
delay = self.rp.get_crawl_delay(to_unicode(user_agent))
return None if delay is None else float(delay)
class ProtegoRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@ -121,3 +138,7 @@ class ProtegoRobotParser(RobotParser):
user_agent = to_unicode(user_agent)
url = to_unicode(url)
return self.rp.can_fetch(url, user_agent)
def crawl_delay(self, user_agent: str | bytes) -> float | None:
delay = self.rp.crawl_delay(to_unicode(user_agent))
return None if delay is None else float(delay)

1
scrapy/signals.py
View File

@ -21,6 +21,7 @@ response_received = object()
response_downloaded = object()
headers_received = object()
bytes_received = object()
robots_parsed = object()
item_scraped = object()
item_dropped = object()
item_error = object()

20
scrapy/utils/asyncio.py
View File

@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar
from twisted.internet.defer import Deferred
from twisted.internet.task import LoopingCall
from twisted.internet.task import LoopingCall, deferLater
from twisted.internet.threads import deferToThread
from scrapy.utils.asyncgen import as_async_generator
@ -293,6 +293,24 @@ class CallLaterResult:
self._delayed_call = None
async def sleep(seconds: float) -> None:
"""Sleep for *seconds*.
.. versionadded:: VERSION
This uses either :func:`asyncio.sleep` or
:func:`~twisted.internet.task.deferLater`, depending on whether asyncio
support is available.
"""
if is_asyncio_available():
await asyncio.sleep(seconds)
return
from twisted.internet import reactor
await deferLater(reactor, seconds)
async def run_in_thread(
func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs
) -> _T:

28
scrapy/utils/console.py
View File

@ -1,8 +1,9 @@
from __future__ import annotations
import asyncio
import code
from collections.abc import Callable
from functools import wraps
from functools import partial, wraps
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
@ -16,16 +17,8 @@ def _embed_ipython_shell(
namespace: dict[str, Any] | None = None, banner: str = ""
) -> EmbedFuncT:
"""Start an IPython Shell"""
try:
from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415
from IPython.terminal.ipapp import load_default_config # noqa: PLC0415
except ImportError:
from IPython.frontend.terminal.embed import ( # type: ignore[import-not-found,no-redef] # noqa: T100,PLC0415
InteractiveShellEmbed,
)
from IPython.frontend.terminal.ipapp import ( # type: ignore[import-not-found,no-redef] # noqa: PLC0415
load_default_config,
)
from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415
from IPython.terminal.ipapp import load_default_config # noqa: PLC0415
@wraps(_embed_ipython_shell)
def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
@ -38,6 +31,19 @@ def _embed_ipython_shell(
shell = InteractiveShellEmbed.instance(
banner1=banner, user_ns=namespace, config=config
)
# If an asyncio event loop is already running in this thread, e.g. when
# inspect_response() is called from a spider callback while using the
# asyncio reactor, prompt_toolkit cannot run its own event loop here, so
# ask it to run the prompt in a separate thread instead. pt_app is None
# when IPython falls back to its simple prompt, which needs no event loop.
# See https://github.com/scrapy/scrapy/issues/5447
if (pt_app := getattr(shell, "pt_app", None)) is not None:
try:
asyncio.get_running_loop()
except RuntimeError:
pass
else:
pt_app.prompt = partial(pt_app.prompt, in_thread=True)
shell()
return wrapper

11
scrapy/utils/defer.py
View File

@ -27,7 +27,7 @@ from twisted.internet.task import Cooperator
from twisted.python import failure
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.asyncio import is_asyncio_available, sleep
from scrapy.utils.python import global_object_name
if TYPE_CHECKING:
@ -90,14 +90,7 @@ async def _defer_sleep_async() -> None:
"""Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers
before attending pending delayed calls, so do not set delay to zero.
"""
if is_asyncio_available():
await asyncio.sleep(_DEFER_DELAY)
else:
from twisted.internet import reactor
d: Deferred[None] = Deferred()
reactor.callLater(_DEFER_DELAY, d.callback, None)
await d
await sleep(_DEFER_DELAY)
def defer_result(result: Any) -> Deferred[Any]: # pragma: no cover

5
scrapy/utils/misc.py
View File

@ -90,6 +90,11 @@ def load_object(path: str | Callable[..., Any]) -> Any:
return obj
def _load_objects(objects: Iterable[str | Callable[..., Any]]) -> tuple[Any, ...]:
"""Resolve *objects* (objects or import paths) to a tuple of objects."""
return tuple(load_object(obj) if isinstance(obj, str) else obj for obj in objects)
def walk_modules_iter(path: str) -> Iterable[ModuleType]:
"""Loads a module and all its submodules from the given module path and
returns them. If *any* module throws an exception while importing, that

28
tests/benchmarks/__init__.py Normal file
View File

@ -0,0 +1,28 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from scrapy.utils.test import get_crawler
if TYPE_CHECKING:
from scrapy import Spider
from scrapy.crawler import Crawler
def crawl(spidercls: type[Spider], settings: dict[str, Any], **kwargs: Any) -> Crawler:
"""Run a crawl to completion and return its crawler.
Unlike the rest of the test suite, benchmarks run without ``pytest-twisted``
and drive the reactor themselves, since the code being measured must be
callable synchronously by ``pytest-codspeed``.
"""
from twisted.internet import reactor
crawler = get_crawler(spidercls, settings)
result: list[Any] = []
crawler.crawl(**kwargs).addBoth(result.append)
while not result:
reactor.iterate(0.001)
if isinstance(result[0], BaseException):
raise result[0]
return crawler

27
tests/benchmarks/conftest.py Normal file
View File

@ -0,0 +1,27 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from scrapy.utils.reactor import install_reactor
if TYPE_CHECKING:
from collections.abc import Generator
@pytest.fixture(scope="session", autouse=True)
def running_reactor() -> Generator[None]:
install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
from twisted.internet import reactor
# Marks the reactor as running without blocking, so that crawls can be
# driven with reactor.iterate(), see tests.benchmarks.crawl().
reactor.startRunning(installSignalHandlers=False)
yield
reactor.stop()
# Lets the shutdown event triggers run, e.g. to join the thread pool.
reactor.iterate(0)

69
tests/benchmarks/test_crawl.py Normal file
View File

@ -0,0 +1,69 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from urllib.parse import urlencode
import pytest
from scrapy import Field, Item, Request, Spider
from scrapy.linkextractors import LinkExtractor
from tests.benchmarks import crawl
if TYPE_CHECKING:
from collections.abc import AsyncIterator
from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]
from scrapy.http import Response
from tests.mockserver.http import MockServer
pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")
PAGES = 100
LINKS_PER_PAGE = 5
class _Page(Item):
url = Field()
anchors = Field()
class _FollowSpider(Spider):
name = "benchmark"
url: str
link_extractor = LinkExtractor()
async def start(self) -> AsyncIterator[Any]:
yield Request(self.url, dont_filter=True)
def parse(self, response: Response) -> Any:
yield _Page(
url=response.url,
anchors=response.css("a::text").getall(),
)
for link in self.link_extractor.extract_links(response): # type: ignore[arg-type]
yield Request(link.url)
class _Pipeline:
def process_item(self, item: Any) -> Any:
return item
def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None:
"""Per-request overhead of a crawl over HTTP.
The pages are small on purpose, so that the cost of parsing them stays
negligible next to the cost of moving requests and responses through the
engine, the middlewares and the download handler.
"""
query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"})
url = mockserver.url(f"/follow?{query}")
settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False}
def run() -> None:
crawler = crawl(_FollowSpider, settings, url=url)
assert crawler.stats
assert crawler.stats.get_value("item_scraped_count") == PAGES + 1
benchmark(run)

39
tests/mockserver/dns.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import sys
from subprocess import PIPE, Popen
from typing import TYPE_CHECKING
from twisted.internet import defer
from twisted.names import dns, error
@ -9,39 +10,63 @@ from twisted.names.server import DNSServerFactory
from tests.utils import get_script_run_env
if TYPE_CHECKING:
from collections.abc import Sequence
from types import TracebackType
from twisted.internet.defer import Deferred
# typing.Self requires Python 3.11
from typing_extensions import Self
_Answers = tuple[list[dns.RRHeader], list[dns.RRHeader], list[dns.RRHeader]]
class MockDNSResolver:
"""
Implements twisted.internet.interfaces.IResolver partially
"""
def _resolve(self, name):
def _resolve(self, name: bytes) -> _Answers:
record = dns.Record_A(address=b"127.0.0.1")
answer = dns.RRHeader(name=name, payload=record)
# zope.interface has no type hints, so mypy cannot tell that Record_A
# provides the IEncodableRecord interface.
answer = dns.RRHeader(name=name, payload=record) # type: ignore[arg-type]
return [answer], [], []
def query(self, query, timeout=None):
def query(
self, query: dns.Query, timeout: Sequence[int] | None = None
) -> Deferred[_Answers]:
if query.type == dns.A:
return defer.succeed(self._resolve(query.name.name))
return defer.fail(error.DomainError())
def lookupAllRecords(self, name, timeout=None):
def lookupAllRecords(
self, name: bytes, timeout: Sequence[int] | None = None
) -> Deferred[_Answers]:
return defer.succeed(self._resolve(name))
class MockDNSServer:
def __enter__(self):
def __enter__(self) -> Self:
self.proc = Popen(
[sys.executable, "-u", "-m", "tests.mockserver.dns"],
stdout=PIPE,
env=get_script_run_env(),
text=True,
)
assert self.proc.stdout is not None
self.host = "127.0.0.1"
self.port = int(self.proc.stdout.readline().strip().split(":")[1])
return self
def __exit__(self, exc_type, exc_value, traceback):
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
self.proc.kill()
self.proc.communicate()
@ -54,7 +79,7 @@ def main() -> None:
protocol = dns.DNSDatagramProtocol(controller=factory)
listener = reactor.listenUDP(0, protocol)
def print_listening():
def print_listening() -> None:
host = listener.getHost()
print(f"{host.host}:{host.port}")

20
tests/mockserver/ftp.py
View File

@ -7,6 +7,7 @@ from pathlib import Path
from shutil import rmtree
from subprocess import PIPE, Popen
from tempfile import mkdtemp
from typing import TYPE_CHECKING
from pyftpdlib.authorizers import DummyAuthorizer
from pyftpdlib.handlers import FTPHandler
@ -14,6 +15,12 @@ from pyftpdlib.servers import FTPServer
from tests.utils import get_script_run_env
if TYPE_CHECKING:
from types import TracebackType
# typing.Self requires Python 3.11
from typing_extensions import Self
class MockFTPServer:
"""Creates an FTP server on a random port with a default passwordless user
@ -26,7 +33,7 @@ class MockFTPServer:
self.port: int | None = None
self.path: Path | None = None
def __enter__(self):
def __enter__(self) -> Self:
self.path = Path(mkdtemp())
self.proc = Popen(
[sys.executable, "-u", "-m", "tests.mockserver.ftp", "-d", str(self.path)],
@ -34,6 +41,7 @@ class MockFTPServer:
env=get_script_run_env(),
text=True,
)
assert self.proc.stderr is not None
for line in self.proc.stderr:
if "starting FTP server" in line and (
m := re.search(r"starting FTP server on ([^ :]+):(\d+),", line)
@ -48,12 +56,18 @@ class MockFTPServer:
)
return self
def __exit__(self, exc_type, exc_value, traceback):
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
rmtree(str(self.path))
assert self.proc is not None
self.proc.kill()
self.proc.communicate()
def url(self, path):
def url(self, path: str) -> str:
return f"ftp://{self.host}:{self.port}/{path}"

85
tests/mockserver/http.py
View File

@ -1,8 +1,8 @@
from __future__ import annotations
from pathlib import Path
from typing import TYPE_CHECKING
from twisted.web import resource
from twisted.web.static import Data, File
from twisted.web.util import Redirect
@ -11,6 +11,7 @@ from tests import tests_datadir
from .http_base import BaseMockServer, main_factory
from .http_resources import (
ArbitraryLengthPayloadResource,
BaseResource,
BrokenChunkedResource,
BrokenDownloadResource,
ChunkedResource,
@ -35,62 +36,68 @@ from .http_resources import (
SetCookie,
Status,
UriResource,
put_child,
)
if TYPE_CHECKING:
from twisted.web.server import Request
class Root(resource.Resource):
def __init__(self):
class Root(BaseResource):
def __init__(self) -> None:
super().__init__()
self.putChild(b"status", Status())
self.putChild(b"follow", Follow())
self.putChild(b"delay", Delay())
self.putChild(b"partial", Partial())
self.putChild(b"drop", Drop())
self.putChild(b"raw", Raw())
self.putChild(b"echo", Echo())
self.putChild(b"payload", PayloadResource())
self.putChild(b"alpayload", ArbitraryLengthPayloadResource())
self.putChild(b"static", File(str(Path(tests_datadir, "test_site/"))))
self.putChild(b"redirect-to", RedirectTo())
self.putChild(b"text", Data(b"Works", "text/plain"))
self.putChild(
put_child(self, b"status", Status())
put_child(self, b"follow", Follow())
put_child(self, b"delay", Delay())
put_child(self, b"partial", Partial())
put_child(self, b"drop", Drop())
put_child(self, b"raw", Raw())
put_child(self, b"echo", Echo())
put_child(self, b"payload", PayloadResource())
put_child(self, b"alpayload", ArbitraryLengthPayloadResource())
put_child(self, b"static", File(str(Path(tests_datadir, "test_site/"))))
put_child(self, b"redirect-to", RedirectTo())
put_child(self, b"text", Data(b"Works", "text/plain"))
put_child(
self,
b"html",
Data(
b"<body><p class='one'>Works</p><p class='two'>World</p></body>",
"text/html",
),
)
self.putChild(
put_child(
self,
b"enc-gb18030",
Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"),
)
self.putChild(b"redirect", Redirect(b"/redirected"))
self.putChild(
b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")
put_child(self, b"redirect", Redirect(b"/redirected"))
put_child(
self, b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")
)
self.putChild(b"redirected", Data(b"Redirected here", "text/plain"))
put_child(self, b"redirected", Data(b"Redirected here", "text/plain"))
numbers = [str(x).encode("utf8") for x in range(2**18)]
self.putChild(b"numbers", Data(b"".join(numbers), "text/plain"))
self.putChild(b"wait", ForeverTakingResource())
self.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
self.putChild(b"host", HostHeaderResource())
self.putChild(b"client-ip", ClientIPResource())
self.putChild(b"broken", BrokenDownloadResource())
self.putChild(b"chunked", ChunkedResource())
self.putChild(b"broken-chunked", BrokenChunkedResource())
self.putChild(b"contentlength", ContentLengthHeaderResource())
self.putChild(b"nocontenttype", EmptyContentTypeHeaderResource())
self.putChild(b"largechunkedfile", LargeChunkedFileResource())
self.putChild(b"compress", Compress())
self.putChild(b"duplicate-header", DuplicateHeaderResource())
self.putChild(b"response-headers", ResponseHeadersResource())
self.putChild(b"set-cookie", SetCookie())
self.putChild(b"uri", UriResource())
put_child(self, b"numbers", Data(b"".join(numbers), "text/plain"))
put_child(self, b"wait", ForeverTakingResource())
put_child(self, b"hang-after-headers", ForeverTakingResource(write=True))
put_child(self, b"host", HostHeaderResource())
put_child(self, b"client-ip", ClientIPResource())
put_child(self, b"broken", BrokenDownloadResource())
put_child(self, b"chunked", ChunkedResource())
put_child(self, b"broken-chunked", BrokenChunkedResource())
put_child(self, b"contentlength", ContentLengthHeaderResource())
put_child(self, b"nocontenttype", EmptyContentTypeHeaderResource())
put_child(self, b"largechunkedfile", LargeChunkedFileResource())
put_child(self, b"compress", Compress())
put_child(self, b"duplicate-header", DuplicateHeaderResource())
put_child(self, b"response-headers", ResponseHeadersResource())
put_child(self, b"set-cookie", SetCookie())
put_child(self, b"uri", UriResource())
def getChild(self, path, request):
def getChild(self, path: bytes, request: Request) -> Root:
return self
def render(self, request):
def render(self, request: Request) -> bytes:
return b"Scrapy mock HTTP server\n"

10
tests/mockserver/http_base.py
View File

@ -17,6 +17,7 @@ from .utils import ssl_context_factory
if TYPE_CHECKING:
from collections.abc import Callable
from types import TracebackType
from twisted.web import resource
@ -60,7 +61,12 @@ class BaseMockServer(ABC):
self.https_port = https_parsed.port
return self
def __exit__(self, exc_type, exc_value, traceback) -> None:
def __exit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: TracebackType | None,
) -> None:
if self.proc:
self.proc.kill()
self.proc.communicate()
@ -135,7 +141,7 @@ def main_factory(
context_factory = ssl_context_factory(**context_factory_kw)
https_port = reactor.listenSSL(0, factory, context_factory)
def print_listening():
def print_listening() -> None:
if listen_http:
http_host = http_port.getHost()
http_address = f"http://{http_host.host}:{http_host.port}"

174
tests/mockserver/http_resources.py
View File

@ -3,7 +3,7 @@ from __future__ import annotations
import gzip
import json
import random
from typing import TYPE_CHECKING, ParamSpec, TypeVar
from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar
from urllib.parse import urlencode
from twisted.internet.task import deferLater
@ -14,17 +14,24 @@ from twisted.web.util import Redirect, redirectTo
from scrapy.utils.python import to_bytes, to_unicode
if TYPE_CHECKING:
from collections.abc import Callable
from collections.abc import Callable, Sequence
from twisted.internet.defer import Deferred
from twisted.web.http import Request
from twisted.python.failure import Failure
from twisted.web.http import Request as HTTPRequest
from twisted.web.server import Request
_T = TypeVar("_T")
_P = ParamSpec("_P")
def getarg(request, name, default=None, type_=None):
def getarg(
request: Request,
name: bytes,
default: Any = None,
type_: Callable[[bytes], Any] | None = None,
) -> Any:
if name in request.args:
value = request.args[name][0]
if type_ is not None:
@ -33,73 +40,91 @@ def getarg(request, name, default=None, type_=None):
return default
def close_connection(request):
def close_connection(request: Request) -> None:
# We have to force a disconnection for HTTP/1.1 clients. Otherwise
# client keeps the connection open waiting for more data.
request.channel.loseConnection()
request.finish()
def put_child(parent: resource.Resource, path: bytes, child: resource.Resource) -> None:
# zope.interface has no type hints, so mypy cannot tell that Resource
# instances provide the IResource interface that putChild() expects.
parent.putChild(path, child) # type: ignore[arg-type]
class BaseResource(resource.Resource):
"""Base class for mockserver resources, with type hints."""
# Only needed to give subclasses a typed __init__ to call.
def __init__(self) -> None: # pylint: disable=useless-parent-delegation
super().__init__() # type: ignore[no-untyped-call]
# most of the following resources are copied from twisted.web.test.test_webclient
class ForeverTakingResource(resource.Resource):
class ForeverTakingResource(BaseResource):
"""
L{ForeverTakingResource} is a resource which never finishes responding
to requests.
"""
def __init__(self, write=False):
resource.Resource.__init__(self)
def __init__(self, write: bool = False):
super().__init__()
self._write = write
def render(self, request):
def render(self, request: Request) -> int:
if self._write:
request.write(b"some bytes")
return server.NOT_DONE_YET
class HostHeaderResource(resource.Resource):
class HostHeaderResource(BaseResource):
"""
A testing resource which renders itself as the value of the host header
from the request.
"""
def render(self, request):
return request.requestHeaders.getRawHeaders(b"host")[0]
def render(self, request: Request) -> bytes:
headers = request.requestHeaders.getRawHeaders(b"host")
assert headers
return headers[0]
class ClientIPResource(resource.Resource):
class ClientIPResource(BaseResource):
"""
A testing resource which renders itself as the request client IP address.
"""
def render(self, request):
def render(self, request: Request) -> bytes:
client_address = request.getClientAddress()
if client_address is None or client_address.host is None:
return b""
return to_bytes(client_address.host)
class PayloadResource(resource.Resource):
class PayloadResource(BaseResource):
"""
A testing resource which renders itself as the contents of the request body
as long as the request body is 100 bytes long, otherwise which renders
itself as C{"ERROR"}.
"""
def render(self, request):
data = request.content.read()
contentLength = request.requestHeaders.getRawHeaders(b"content-length")[0]
if len(data) != 100 or int(contentLength) != 100:
def render(self, request: Request) -> bytes:
assert request.content
data: bytes = request.content.read()
content_length = request.requestHeaders.getRawHeaders(b"content-length")
assert content_length
if len(data) != 100 or int(content_length[0]) != 100:
return b"ERROR"
return data
class LeafResource(resource.Resource):
class LeafResource(BaseResource):
isLeaf = True
def deferRequest(
self,
request: Request,
request: HTTPRequest,
delay: float,
f: Callable[_P, _T],
*a: _P.args,
@ -107,7 +132,7 @@ class LeafResource(resource.Resource):
) -> Deferred[_T]:
from twisted.internet import reactor
def _cancelrequest(_):
def _cancelrequest(_: Failure) -> None:
# silence CancelledError
d.addErrback(lambda _: None)
d.cancel()
@ -118,12 +143,13 @@ class LeafResource(resource.Resource):
class Follow(LeafResource):
def render(self, request):
def render(self, request: Request) -> int:
total = getarg(request, b"total", 100, type_=int)
show = getarg(request, b"show", 1, type_=int)
order = getarg(request, b"order", b"desc")
maxlatency = getarg(request, b"maxlatency", 0, type_=float)
n = getarg(request, b"n", total, type_=int)
nlist: Sequence[int]
if order == b"rand":
nlist = [random.randint(1, total) for _ in range(show)]
else: # order == "desc"
@ -133,7 +159,7 @@ class Follow(LeafResource):
self.deferRequest(request, lag, self.renderRequest, request, nlist)
return NOT_DONE_YET
def renderRequest(self, request, nlist):
def renderRequest(self, request: Request, nlist: Sequence[int]) -> None:
s = """<html> <head></head> <body>"""
args = request.args.copy()
for nl in nlist:
@ -146,45 +172,47 @@ class Follow(LeafResource):
class Delay(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> int:
n = getarg(request, b"n", 1, type_=float)
b = getarg(request, b"b", 1, type_=int)
if b:
# send headers now and delay body
request.write("")
request.write(b"")
self.deferRequest(request, n, self._delayedRender, request, n)
return NOT_DONE_YET
def _delayedRender(self, request, n):
def _delayedRender(self, request: Request, n: float) -> None:
request.write(to_bytes(f"Response delayed for {n:.3f} seconds\n"))
request.finish()
class Status(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> bytes:
n = getarg(request, b"n", 200, type_=int)
request.setResponseCode(n)
return b""
class Raw(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> int:
request.startedWriting = 1
self.deferRequest(request, 0, self._delayedRender, request)
return NOT_DONE_YET
render_POST = render_GET
def _delayedRender(self, request):
def _delayedRender(self, request: Request) -> None:
raw = getarg(request, b"raw", b"HTTP 1.1 200 OK\n")
request.startedWriting = 1
request.write(raw)
assert request.channel.transport is not None
request.channel.transport.loseConnection()
request.finish()
class Echo(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> bytes:
assert request.content
output = {
"headers": {
to_unicode(k): [to_unicode(v) for v in vs]
@ -198,27 +226,29 @@ class Echo(LeafResource):
class RedirectTo(LeafResource):
def render(self, request):
def render(self, request: Request) -> bytes:
goto = getarg(request, b"goto", b"/")
# we force the body content, otherwise Twisted redirectTo()
# returns HTML with <meta http-equiv="refresh"
redirectTo(goto, request)
# zope.interface has no type hints, so mypy cannot tell that Request
# provides the IRequest interface.
redirectTo(goto, request) # type: ignore[arg-type]
return b"redirecting..."
class Partial(LeafResource):
def render_GET(self, request):
def render_GET(self, request: Request) -> int:
request.setHeader(b"Content-Length", b"1024")
self.deferRequest(request, 0, self._delayedRender, request)
return NOT_DONE_YET
def _delayedRender(self, request):
def _delayedRender(self, request: Request) -> None:
request.write(b"partial content\n")
request.finish()
class Drop(Partial):
def _delayedRender(self, request):
def _delayedRender(self, request: Request) -> None:
abort = getarg(request, b"abort", 0, type_=int)
request.write(b"this connection will be dropped\n")
tr = request.channel.transport
@ -233,8 +263,10 @@ class Drop(Partial):
class ArbitraryLengthPayloadResource(LeafResource):
def render(self, request):
return request.content.read()
def render(self, request: Request) -> bytes:
assert request.content
data: bytes = request.content.read()
return data
class NoMetaRefreshRedirect(Redirect):
@ -245,21 +277,23 @@ class NoMetaRefreshRedirect(Redirect):
)
class ContentLengthHeaderResource(resource.Resource):
class ContentLengthHeaderResource(BaseResource):
"""
A testing resource which renders itself as the value of the Content-Length
header from the request.
"""
def render(self, request):
return request.requestHeaders.getRawHeaders(b"content-length")[0]
def render(self, request: Request) -> bytes:
headers = request.requestHeaders.getRawHeaders(b"content-length")
assert headers
return headers[0]
class ChunkedResource(resource.Resource):
def render(self, request):
class ChunkedResource(BaseResource):
def render(self, request: Request) -> int:
from twisted.internet import reactor
def response():
def response() -> None:
request.write(b"chunked ")
request.write(b"content\n")
request.finish()
@ -268,11 +302,11 @@ class ChunkedResource(resource.Resource):
return server.NOT_DONE_YET
class BrokenChunkedResource(resource.Resource):
def render(self, request):
class BrokenChunkedResource(BaseResource):
def render(self, request: Request) -> int:
from twisted.internet import reactor
def response():
def response() -> None:
request.write(b"chunked ")
request.write(b"content\n")
# Disable terminating chunk on finish.
@ -283,11 +317,11 @@ class BrokenChunkedResource(resource.Resource):
return server.NOT_DONE_YET
class BrokenDownloadResource(resource.Resource):
def render(self, request):
class BrokenDownloadResource(BaseResource):
def render(self, request: Request) -> int:
from twisted.internet import reactor
def response():
def response() -> None:
request.setHeader(b"Content-Length", b"20")
request.write(b"partial")
close_connection(request)
@ -296,22 +330,24 @@ class BrokenDownloadResource(resource.Resource):
return server.NOT_DONE_YET
class EmptyContentTypeHeaderResource(resource.Resource):
class EmptyContentTypeHeaderResource(BaseResource):
"""
A testing resource which renders itself as the value of request body
without content-type header in response.
"""
def render(self, request):
def render(self, request: Request) -> bytes:
assert request.content
request.setHeader("content-type", "")
return request.content.read()
data: bytes = request.content.read()
return data
class LargeChunkedFileResource(resource.Resource):
def render(self, request):
class LargeChunkedFileResource(BaseResource):
def render(self, request: Request) -> int:
from twisted.internet import reactor
def response():
def response() -> None:
for _ in range(1024):
request.write(b"x" * 1024)
request.finish()
@ -320,43 +356,45 @@ class LargeChunkedFileResource(resource.Resource):
return server.NOT_DONE_YET
class DuplicateHeaderResource(resource.Resource):
def render(self, request):
class DuplicateHeaderResource(BaseResource):
def render(self, request: Request) -> bytes:
request.responseHeaders.setRawHeaders(b"Set-Cookie", [b"a=b", b"c=d"])
return b""
class UriResource(resource.Resource):
class UriResource(BaseResource):
"""Return the full uri that was requested"""
def getChild(self, path, request):
def getChild(self, path: bytes, request: Request) -> resource.Resource:
return self
def render(self, request):
def render(self, request: Request) -> bytes | int:
# Note: this is an ugly hack for CONNECT request timeout test.
# Returning some data here fail SSL/TLS handshake
# ToDo: implement proper HTTPS proxy tests, not faking them.
if request.method != b"CONNECT":
return request.uri
assert request.transport is not None
request.transport.write(b"HTTP/1.1 200 Connection established\r\n\r\n")
return NOT_DONE_YET
class ResponseHeadersResource(resource.Resource):
class ResponseHeadersResource(BaseResource):
"""Return a response with headers set from the JSON request body"""
def render(self, request):
def render(self, request: Request) -> bytes:
assert request.content
body = json.loads(request.content.read().decode())
for header_name, header_value in body.items():
request.responseHeaders.setRawHeaders(header_name, [header_value])
return json.dumps(body).encode("utf-8")
class Compress(resource.Resource):
class Compress(BaseResource):
"""Compress the data sent in the request url params and set Content-Encoding header"""
def render(self, request):
data = request.args.get(b"data")[0]
def render(self, request: Request) -> bytes:
data = request.args[b"data"][0]
accept_encoding_header = request.getHeader(b"accept-encoding")
@ -370,10 +408,10 @@ class Compress(resource.Resource):
return b"Did not receive a valid accept-encoding header"
class SetCookie(resource.Resource):
class SetCookie(BaseResource):
"""Return a response with a Set-Cookie header for each request url parameter"""
def render(self, request):
def render(self, request: Request) -> bytes:
for cookie_name, cookie_values in request.args.items():
for cookie_value in cookie_values:
cookie = (cookie_name.decode() + "=" + cookie_value.decode()).encode()

19
tests/mockserver/simple_https.py
View File

@ -2,18 +2,23 @@
from __future__ import annotations
from twisted.web import resource
from typing import TYPE_CHECKING
from twisted.web.static import Data
from .http_base import BaseMockServer, main_factory
from .http_resources import BaseResource, put_child
if TYPE_CHECKING:
from twisted.web.server import Request
class Root(resource.Resource):
def __init__(self):
resource.Resource.__init__(self)
self.putChild(b"file", Data(b"0123456789", "text/plain"))
class Root(BaseResource):
def __init__(self) -> None:
super().__init__()
put_child(self, b"file", Data(b"0123456789", "text/plain"))
def getChild(self, path, request):
def getChild(self, path: bytes, request: Request) -> Root:
return self
@ -29,7 +34,7 @@ class SimpleMockServer(BaseMockServer):
cipher_string: str | None = None,
tls_min_version: str | None = None,
tls_max_version: str | None = None,
):
) -> None:
super().__init__()
self.keyfile = keyfile
self.certfile = certfile

12
tests/test_command_crawl.py
View File

@ -23,6 +23,18 @@ class TestCrawlCommand(TestProjectBase):
_, _, stderr = self.crawl(code, proj_path, args=args)
return stderr
def test_no_spider(self, proj_path: Path) -> None:
returncode, out, _ = proc("crawl", cwd=proj_path)
assert returncode == 2
assert "Usage" in out
def test_multiple_spiders(self, proj_path: Path) -> None:
returncode, _, err = proc("crawl", "myspider", "myspider2", cwd=proj_path)
assert returncode == 2
assert (
"running 'scrapy crawl' with more than one spider is not supported" in err
)
def test_no_output(self, proj_path: Path) -> None:
spider_code = """
import scrapy

32
tests/test_command_fetch.py
View File

@ -2,13 +2,24 @@ from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from tests.utils.bases.commands import TestProjectBase
from tests.utils.cmdline import proc
if TYPE_CHECKING:
from pathlib import Path
from tests.mockserver.http import MockServer
class TestFetchCommand:
@pytest.mark.parametrize("args", [(), ("not-a-url",), ("a:b", "c:d")])
def test_bad_arguments(self, args: tuple[str, ...]) -> None:
returncode, out, _ = proc("fetch", *args)
assert returncode == 2
assert "Usage" in out
def test_output(self, mockserver: MockServer) -> None:
_, out, _ = proc("fetch", mockserver.url("/text"))
assert out.strip() == "Works"
@ -36,3 +47,24 @@ class TestFetchCommand:
"fetch", "-s", "TWISTED_REACTOR_ENABLED=False", mockserver.url("/text")
)
assert out.strip() == "Works"
class TestFetchCommandWithSpider(TestProjectBase):
@pytest.fixture(autouse=True)
def create_files(self, proj_path: Path) -> None:
(proj_path / self.project_name / "spiders" / "myspider.py").write_text(
"""
import scrapy
class MySpider(scrapy.Spider):
name = "myspider"
custom_settings = {"USER_AGENT": "myspider-user-agent"}
""",
encoding="utf-8",
)
def test_spider(self, proj_path: Path, mockserver: MockServer) -> None:
_, out, err = proc(
"fetch", "--spider", "myspider", mockserver.url("/echo"), cwd=proj_path
)
assert "myspider-user-agent" in out, err

21
tests/test_command_genspider.py
View File

@ -64,6 +64,24 @@ class TestGenspiderCommand(TestProjectBase):
assert call("genspider", "--dump=basic", cwd=proj_path) == 0
assert call("genspider", "-d", "basic", cwd=proj_path) == 0
@pytest.mark.parametrize(
"args",
[("--dump=nonexistent",), ("-t", "nonexistent", "test_name", "test.com")],
)
def test_unknown_template(self, args: tuple[str, ...], proj_path: Path) -> None:
returncode, out, err = proc("genspider", *args, cwd=proj_path)
assert returncode == 0, err
assert "Unable to find template: nonexistent" in out
assert not (proj_path / self.project_name / "spiders" / "test_name.py").exists()
def test_name_not_starting_with_a_letter(self, proj_path: Path) -> None:
"""The module name, unlike the spider name, is prefixed with a letter."""
_, out, err = proc("genspider", "1st_spider", "test.com", cwd=proj_path)
assert "Created spider '1st_spider'" in out, err
spider = proj_path / self.project_name / "spiders" / "a1st_spider.py"
assert spider.exists()
assert find_in_file(spider, r'name\s*=\s*"1st_spider"') is not None
@pytest.mark.skipif(
sys.platform == "win32", reason="requires a POSIX shell editor script"
)
@ -87,7 +105,8 @@ class TestGenspiderCommand(TestProjectBase):
)
def test_same_name_as_project(self, proj_path: Path) -> None:
assert call("genspider", self.project_name, cwd=proj_path) == 2
_, out, err = proc("genspider", self.project_name, "test.com", cwd=proj_path)
assert "Cannot create a spider with the same name as your project" in out, err
assert not (
proj_path / self.project_name / "spiders" / f"{self.project_name}.py"
).exists()

125
tests/test_command_parse.py
View File

@ -3,6 +3,7 @@ from __future__ import annotations
import argparse
import re
from typing import TYPE_CHECKING
from urllib.parse import urlparse
import pytest
@ -552,6 +553,130 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
content = '[\n{},\n{"foo": "bar"}\n]'
assert file_path.read_text(encoding="utf-8") == content
@pytest.mark.parametrize("args", [(), ("not-a-url",), ("a:b", "c:d")])
def test_bad_arguments(self, args: tuple[str, ...], proj_path: Path) -> None:
returncode, out, _ = proc("parse", *args, cwd=proj_path)
assert returncode == 2
assert "Usage" in out
@pytest.mark.parametrize(
("option", "message"),
[
("--meta", "Invalid -m/--meta value"),
("-m", "Invalid -m/--meta value"),
("--cbkwargs", "Invalid --cbkwargs value"),
],
)
def test_invalid_json(
self, option: str, message: str, proj_path: Path, mockserver: MockServer
) -> None:
returncode, _, err = proc(
"parse",
"--spider",
self.spider_name,
option,
"{invalid",
mockserver.url("/html"),
cwd=proj_path,
)
assert returncode == 2
assert message in err
def test_unknown_spider(self, proj_path: Path, mockserver: MockServer) -> None:
returncode, _, err = proc(
"parse",
"--spider",
"nonexistent",
mockserver.url("/html"),
cwd=proj_path,
)
assert returncode == 0, err
assert "Unable to find spider: nonexistent" in err
def test_spider_found_by_url(self, proj_path: Path, mockserver: MockServer) -> None:
"""Without --spider, the spider is chosen based on the URL."""
url = mockserver.url("/html")
# The spider name doubles as a domain of the spider, and it is matched
# against the netloc of the URL, hence the port.
(proj_path / self.project_name / "spiders" / "urlspider.py").write_text(
f"""
import scrapy
class UrlSpider(scrapy.Spider):
name = "{urlparse(url).netloc}"
def parse(self, response):
return [{{"found_by_url": True}}]
""",
encoding="utf-8",
)
returncode, out, err = proc("parse", url, cwd=proj_path)
assert returncode == 0, err
assert "Unable to find spider for" not in err
assert "{'found_by_url': True}" in out
def test_legacy_item_processor(
self, proj_path: Path, mockserver: MockServer
) -> None:
"""--pipelines supports an ITEM_PROCESSOR without process_item_async()."""
(proj_path / self.project_name / "legacy.py").write_text(
"""
import logging
from twisted.internet.defer import succeed
class LegacyItemProcessor:
@classmethod
def from_crawler(cls, crawler):
return cls()
def open_spider(self, spider):
return succeed(None)
def close_spider(self, spider):
return succeed(None)
def process_item(self, item, spider):
logging.info("Legacy item processor!")
return succeed(item)
""",
encoding="utf-8",
)
_, _, stderr = proc(
"parse",
"--spider",
self.spider_name,
"--pipelines",
"-c",
"parse",
"-s",
f"ITEM_PROCESSOR={self.project_name}.legacy.LegacyItemProcessor",
mockserver.url("/html"),
cwd=proj_path,
)
assert "INFO: Legacy item processor!" in stderr
@pytest.mark.parametrize("verbose", [True, False])
def test_no_items_no_links(
self, verbose: bool, proj_path: Path, mockserver: MockServer
) -> None:
args = ["--verbose"] if verbose else []
_, out, err = proc(
"parse",
"--spider",
self.spider_name,
"-c",
"parse",
"--noitems",
"--nolinks",
*args,
mockserver.url("/html"),
cwd=proj_path,
)
assert "# Scraped Items" not in out, err
assert "# Requests" not in out
def test_parse_add_options(self):
command = parse.Command()
command.settings = Settings()

6
tests/test_command_runspider.py
View File

@ -136,6 +136,12 @@ class MySpider(scrapy.Spider):
log = self.get_log(tmp_path, "from scrapy.spiders import Spider\n")
assert "No spider found in file" in log
@pytest.mark.parametrize("args", [(), ("a.py", "b.py")])
def test_runspider_bad_arguments(self, args: tuple[str, ...]) -> None:
returncode, out, _ = proc("runspider", *args)
assert returncode == 2
assert "Usage" in out
def test_runspider_file_not_found(self) -> None:
_, _, log = proc("runspider", "some_non_existent_file")
assert "File not found: some_non_existent_file" in log

28
tests/test_command_shell.py
View File

@ -18,6 +18,7 @@ from scrapy.shell import Shell, inspect_response
from scrapy.utils.reactor import _asyncio_reactor_path
from scrapy.utils.test import get_crawler
from tests import NON_EXISTING_RESOLVABLE, tests_datadir
from tests.utils.bases.commands import TestProjectBase
from tests.utils.cmdline import proc
from tests.utils.decorators import coroutine_test
@ -162,6 +163,33 @@ class TestShellCommand:
assert ret == 0, out
class TestShellCommandWithSpider(TestProjectBase):
@pytest.fixture(autouse=True)
def create_files(self, proj_path: Path) -> None:
(proj_path / self.project_name / "spiders" / "myspider.py").write_text(
"""
import scrapy
class MySpider(scrapy.Spider):
name = "myspider"
""",
encoding="utf-8",
)
def test_spider(self, proj_path: Path, mockserver: MockServer) -> None:
ret, out, err = proc(
"shell",
"--spider",
"myspider",
mockserver.url("/text"),
"-c",
"spider.name",
cwd=proj_path,
)
assert ret == 0, err
assert out.strip() == "myspider"
class TestInteractiveShell:
def test_fetch(self, mockserver: MockServer) -> None:
args = (

139
tests/test_commands.py
View File

@ -3,21 +3,27 @@ from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import TYPE_CHECKING
import pytest
import scrapy
from scrapy.cmdline import _pop_command_name, execute
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.settings import Settings
from scrapy.utils.reactor import _asyncio_reactor_path
from tests.utils.bases.commands import TestProjectBase
from tests.utils.cmdline import call, proc, write_recording_editor
from tests.utils.cmdline import (
call,
proc,
write_recording_browser,
write_recording_editor,
)
if TYPE_CHECKING:
from pathlib import Path
from tests.mockserver.http import MockServer
class EmptyCommand(ScrapyCommand):
@ -107,6 +113,93 @@ class TestCommandSettings:
)
class TestGlobalOptions:
"""Tests for the options that every command supports."""
spider_code = """
import scrapy
class MySpider(scrapy.Spider):
name = "myspider"
async def start(self):
self.logger.debug("It works!")
return
yield
"""
@pytest.fixture
def spider_path(self, tmp_path: Path) -> Path:
path = tmp_path / "myspider.py"
path.write_text(self.spider_code, encoding="utf-8")
return path
def test_invalid_set(self, spider_path: Path) -> None:
returncode, _, err = proc("runspider", str(spider_path), "-s", "FOO")
assert returncode == 2
assert "Invalid -s value, use -s NAME=VALUE" in err
def test_invalid_spider_argument(self, spider_path: Path) -> None:
returncode, _, err = proc("runspider", str(spider_path), "-a", "FOO")
assert returncode == 2
assert "Invalid -a value, use -a NAME=VALUE" in err
def test_logfile(self, tmp_path: Path, spider_path: Path) -> None:
logfile = tmp_path / "scrapy.log"
returncode, _, err = proc(
"runspider", str(spider_path), "--logfile", str(logfile)
)
assert returncode == 0, err
assert "It works!" in logfile.read_text(encoding="utf-8")
assert "It works!" not in err
def test_loglevel(self, spider_path: Path) -> None:
returncode, _, err = proc("runspider", str(spider_path), "--loglevel", "INFO")
assert returncode == 0, err
assert "It works!" not in err
assert "Spider closed (finished)" in err
def test_nolog(self, spider_path: Path) -> None:
returncode, _, err = proc("runspider", str(spider_path), "--nolog")
assert returncode == 0, err
assert not err
def test_pidfile(self, tmp_path: Path, spider_path: Path) -> None:
pidfile = tmp_path / "scrapy.pid"
returncode, _, err = proc(
"runspider", str(spider_path), "--pidfile", str(pidfile)
)
assert returncode == 0, err
assert pidfile.read_text(encoding="utf-8").strip().isdigit()
def test_pdb(self, spider_path: Path) -> None:
returncode, _, err = proc("runspider", str(spider_path), "--pdb")
assert returncode == 0, err
assert "It works!" in err
class TestSettingsCommand:
@pytest.mark.parametrize(
("option", "setting", "expected"),
[
("--get", "BOT_NAME", "scrapybot"),
("--getbool", "COOKIES_ENABLED", "True"),
("--getint", "CONCURRENT_REQUESTS", "16"),
("--getfloat", "DOWNLOAD_DELAY", "0.0"),
("--getlist", "SPIDER_MODULES", "[]"),
],
)
def test_get(self, option: str, setting: str, expected: str) -> None:
returncode, out, err = proc("settings", option, setting)
assert returncode == 0, err
assert out.startswith(expected)
def test_no_option(self) -> None:
returncode, out, err = proc("settings")
assert returncode == 0, err
assert not out
class TestCommandCrawlerProcess(TestProjectBase):
"""Test that the command uses the expected kind of *CrawlerProcess
and produces expected errors when needed."""
@ -577,18 +670,31 @@ class TestBenchCommand:
class TestViewCommand:
def test_methods(self) -> None:
command = view.Command()
command.settings = Settings()
parser = argparse.ArgumentParser(
prog="scrapy",
prefix_chars="-",
formatter_class=ScrapyHelpFormatter,
conflict_handler="resolve",
@pytest.mark.skipif(
sys.platform == "win32", reason="requires a POSIX shell browser script"
)
def test_view(
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, mockserver: MockServer
) -> None:
opened = tmp_path / "opened.txt"
browser = tmp_path / "fake-browser.sh"
write_recording_browser(browser, opened)
monkeypatch.setenv("BROWSER", str(browser))
returncode, _, err = proc("view", mockserver.url("/html"), cwd=tmp_path)
assert returncode == 0, err
url = opened.read_text(encoding="utf-8")
assert url.startswith("file://")
body = Path(url.removeprefix("file://")).read_text(encoding="utf-8")
assert "<p class='one'>Works</p>" in body
def test_non_text_response(self, mockserver: MockServer) -> None:
returncode, _, err = proc(
"view", mockserver.url("/static/files/images/scrapy.png")
)
command.add_options(parser)
assert command.short_desc() == "Open URL in browser, as seen by Scrapy"
assert "URL using the Scrapy downloader and show its" in command.long_desc()
assert returncode == 0, err
assert "Cannot view a non-text response." in err
class TestEditCommand(TestProjectBase):
@ -615,6 +721,11 @@ class TestEditCommand(TestProjectBase):
assert returncode == 1
assert "Spider not found: nonexistent" in err
def test_edit_no_spider(self, proj_path: Path) -> None:
returncode, out, _ = proc("edit", cwd=proj_path)
assert returncode == 2
assert "Usage" in out
class TestHelpMessage(TestProjectBase):
@pytest.mark.parametrize(

32
tests/test_core_downloader.py
View File

@ -14,6 +14,7 @@ from twisted.web import server, static
from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
from twisted.web.client import Response as TxResponse
from scrapy import Request, Spider
from scrapy.core.downloader import Downloader, Slot, tls
from scrapy.core.downloader.contextfactory import (
_load_context_factory_from_settings,
@ -30,14 +31,17 @@ from scrapy.utils.misc import build_from_crawler
from scrapy.utils.python import to_bytes
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.http_resources import PayloadResource
from tests.mockserver.http_resources import PayloadResource, put_child
from tests.mockserver.utils import ssl_context_factory
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from twisted.internet.defer import Deferred
from twisted.internet.interfaces import IListeningPort
from twisted.web.iweb import IBodyProducer
from scrapy.http import Response
class TestSlot:
def test_repr(self):
@ -51,7 +55,7 @@ class TestContextFactoryBase:
async def server_url(self, tmp_path):
(tmp_path / "file").write_bytes(b"0123456789")
r = static.File(str(tmp_path))
r.putChild(b"payload", PayloadResource())
put_child(r, b"payload", PayloadResource())
site = server.Site(r, timeout=None)
port = self._listen(site)
portno = port.getHost().port
@ -60,7 +64,7 @@ class TestContextFactoryBase:
await port.stopListening()
def _listen(self, site):
def _listen(self, site: server.Site) -> IListeningPort:
from twisted.internet import reactor
return reactor.listenSSL(
@ -296,10 +300,30 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
await self._assert_factory_works(server_url, client_context_factory)
@pytest.mark.parametrize(
("concurrency", "active", "expected"),
[
(2, 1, False),
(2, 2, True),
(0, 0, False),
(0, 2, False),
],
)
def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:
crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})
downloader = Downloader(crawler)
downloader.active = {Request(f"https://example.com/{i}") for i in range(active)}
assert downloader.needs_backout() is expected
downloader.close()
@coroutine_test
async def test_fetch_deprecated_spider_arg():
class CustomDownloader(Downloader):
def fetch(self, request, spider): # pylint: disable=signature-differs
# requiring the spider argument is what triggers the deprecation
def fetch( # type: ignore[override] # pylint: disable=signature-differs
self, request: Request, spider: Spider
) -> Deferred[Response | Request]:
return super().fetch(request, spider)
crawler = get_crawler(DefaultSpider, {"DOWNLOADER": CustomDownloader})

33
tests/test_crawler.py
View File

@ -74,6 +74,39 @@ class TestCrawler:
assert not settings.frozen
assert crawler.settings.frozen
@pytest.mark.parametrize(
("attr", "setting"),
[
("download_delay", "DOWNLOAD_DELAY"),
("max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"),
],
)
def test_deprecated_spider_attr(self, attr: str, setting: str) -> None:
crawler = get_raw_crawler(type("_Spider", (DefaultSpider,), {attr: 2}))
with pytest.warns(
ScrapyDeprecationWarning,
match=f"The {attr!r} spider attribute is deprecated. Use the {setting} ",
):
crawler._apply_settings()
assert crawler.settings.getint(setting) == 2
@pytest.mark.parametrize(
("attr", "setting"),
[
("download_delay", "DOWNLOAD_DELAY"),
("max_concurrent_requests", "CONCURRENT_REQUESTS_PER_DOMAIN"),
],
)
def test_deprecated_spider_attr_ignored(self, attr: str, setting: str) -> None:
crawler = get_raw_crawler(type("_Spider", (DefaultSpider,), {attr: 2}))
crawler.settings.set(setting, 3, priority="spider")
with pytest.warns(
ScrapyDeprecationWarning,
match=f"The {attr!r} spider attribute is deprecated. It is also being ",
):
crawler._apply_settings()
assert crawler.settings.getint(setting) == 3
def test_crawler_accepts_dict(self) -> None:
crawler = get_crawler(DefaultSpider, {"foo": "bar"})
assert crawler.settings["foo"] == "bar"

5
tests/test_crawler_subprocess.py
View File

@ -14,7 +14,8 @@ from packaging.version import parse as parse_version
from pexpect.popen_spawn import PopenSpawn
from w3lib import __version__ as w3lib_version
from tests.utils import async_sleep, get_script_run_env
from scrapy.utils.asyncio import sleep
from tests.utils import get_script_run_env
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
@ -244,7 +245,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
p.kill(sig)
p.expect_exact("shutting down gracefully")
# sending the second signal too fast often causes problems
await async_sleep(0.01)
await sleep(0.01)
p.kill(sig)
p.expect_exact("forcing unclean shutdown")
p.wait() # type: ignore[no-untyped-call]

14
tests/test_downloader_handler_httpx.py
View File

@ -15,6 +15,8 @@ from scrapy.core.downloader.handlers._httpx import (
HttpxDownloadHandler,
)
from scrapy.exceptions import DownloadFailedError
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.bases.download_handlers_http import (
TestHttpBase,
TestHttpProxyBase,
@ -161,3 +163,15 @@ class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
@pytest.mark.requires_internet
class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase):
pass
@pytest.mark.parametrize(("concurrency", "expected"), [(16, 16), (0, None)])
@coroutine_test
async def test_pool_limits(concurrency: int, expected: int | None) -> None:
crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})
handler = build_from_crawler(HttpxDownloadHandler, crawler)
try:
assert handler._limits.max_connections == expected
assert handler._limits.max_keepalive_connections == expected
finally:
await handler.close()

22
tests/test_downloader_handler_twisted_ftp.py
View File

@ -156,14 +156,17 @@ class TestFTP(TestFTPBase):
for filename, content in self.test_files:
(userdir / filename).write_bytes(content)
def _get_factory(self, root):
def _get_factory(self, root: Path) -> FTPFactory:
from twisted.protocols.ftp import FTPFactory, FTPRealm
realm = FTPRealm(anonymousRoot=str(root), userHome=str(root))
p = portal.Portal(realm)
# zope.interface has no type hints, so mypy cannot tell that these
# objects provide the interfaces that Portal expects.
p = portal.Portal(realm) # type: ignore[arg-type]
users_checker = checkers.InMemoryUsernamePasswordDatabaseDontUse()
users_checker.addUser(self.username, self.password)
p.registerChecker(users_checker, credentials.IUsernamePassword)
# the FTP protocol authenticates with str credentials
users_checker.addUser(self.username, self.password) # type: ignore[arg-type]
p.registerChecker(users_checker, credentials.IUsernamePassword) # type: ignore[arg-type]
return FTPFactory(portal=p)
@deferred_f_from_coro_f
@ -192,12 +195,17 @@ class TestAnonymousFTP(TestFTPBase):
for filename, content in self.test_files:
(root / filename).write_bytes(content)
def _get_factory(self, tmp_path):
def _get_factory(self, tmp_path: Path) -> FTPFactory:
from twisted.protocols.ftp import FTPFactory, FTPRealm
realm = FTPRealm(anonymousRoot=str(tmp_path))
p = portal.Portal(realm)
p.registerChecker(checkers.AllowAnonymousAccess(), credentials.IAnonymous)
# zope.interface has no type hints, so mypy cannot tell that these
# objects provide the interfaces that Portal expects.
p = portal.Portal(realm) # type: ignore[arg-type]
p.registerChecker(
checkers.AllowAnonymousAccess(), # type: ignore[arg-type]
credentials.IAnonymous,
)
return FTPFactory(portal=p, userAnonymous=self.username)

66
tests/test_downloadermiddleware_cookies.py
View File

@ -1,5 +1,6 @@
import logging
from collections.abc import Iterable
from typing import Any
import pytest
@ -219,7 +220,7 @@ class TestCookiesMiddleware:
def test_complex_cookies(self):
# merge some cookies into jar
cookies = [
cookies: list[VerboseCookie] = [
{
"name": "C1",
"value": "value1",
@ -483,13 +484,13 @@ class TestCookiesMiddleware:
def _test_cookie_redirect(
self,
source,
target,
source: str | dict[str, Any],
target: str | dict[str, Any],
*,
cookies1,
cookies2,
):
input_cookies = {"a": "b"}
cookies1: bool,
cookies2: bool,
) -> None:
input_cookies: CookiesT = {"a": "b"}
if not isinstance(source, dict):
source = {"url": source}
@ -551,11 +552,11 @@ class TestCookiesMiddleware:
def _test_cookie_header_redirect(
self,
source,
target,
source: str | dict[str, Any],
target: str | dict[str, Any],
*,
cookies2,
):
cookies2: bool,
) -> None:
"""Test the handling of a user-defined Cookie header when building a
redirect follow-up request.
@ -623,14 +624,14 @@ class TestCookiesMiddleware:
def _test_user_set_cookie_domain_followup(
self,
url1,
url2,
domain,
url1: str,
url2: str,
domain: str,
*,
cookies1,
cookies2,
):
input_cookies = [
cookies1: bool,
cookies2: bool,
) -> None:
input_cookies: list[VerboseCookie] = [
{
"name": "a",
"value": "b",
@ -686,16 +687,16 @@ class TestCookiesMiddleware:
def _test_server_set_cookie_domain_followup(
self,
url1,
url2,
domain,
url1: str,
url2: str,
domain: str,
*,
cookies,
):
cookies: bool,
) -> None:
request1 = Request(url1)
self.mw.process_request(request1)
input_cookies = [
input_cookies: list[VerboseCookie] = [
{
"name": "a",
"value": "b",
@ -747,8 +748,14 @@ class TestCookiesMiddleware:
)
def _test_cookie_redirect_scheme_change(
self, secure, from_scheme, to_scheme, cookies1, cookies2, cookies3
):
self,
secure: bool | object,
from_scheme: str,
to_scheme: str,
cookies1: bool,
cookies2: bool,
cookies3: bool,
) -> None:
"""When a redirect causes the URL scheme to change from *from_scheme*
to *to_scheme*, while domain and port remain the same, and given a
cookie on the initial request with its secure attribute set to
@ -756,10 +763,11 @@ class TestCookiesMiddleware:
initial request (*cookies1*), if it should be kept by the redirect
middleware (*cookies2*), and if it should be present on the Cookie
header in the redirected request (*cookie3*)."""
cookie_kwargs = {}
cookie: VerboseCookie = {"name": "a", "value": "b"}
if secure is not UNSET:
cookie_kwargs["secure"] = secure
input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}]
assert isinstance(secure, bool)
cookie["secure"] = secure
input_cookies = [cookie]
request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies)
self.mw.process_request(request1)

8
tests/test_downloadermiddleware_httpauth.py
View File

@ -1,3 +1,5 @@
from typing import Any
import pytest
from w3lib.http import basic_auth_header
@ -10,8 +12,10 @@ from scrapy.utils.test import get_crawler
_DOMAIN_NOT_SET = object()
def make_mw(user="", passwd="", domain=_DOMAIN_NOT_SET):
settings: dict = {
def make_mw(
user: str = "", passwd: str = "", domain: str | object = _DOMAIN_NOT_SET
) -> HttpAuthMiddleware:
settings: dict[str, Any] = {
"HTTPAUTH_USER": user,
"HTTPAUTH_PASS": passwd,
}

31
tests/test_downloadermiddleware_httpcache.py
View File

@ -94,14 +94,14 @@ class TestBase:
finally:
mw.spider_closed(crawler.spider)
def assertEqualResponse(self, response1, response2):
def assertEqualResponse(self, response1: Response, response2: Response) -> None:
assert response1.url == response2.url
assert response1.status == response2.status
assert response1.headers == response2.headers
assert response1.body == response2.body
class StorageTestMixin:
class StorageTestMixin(TestBase):
"""Mixin containing storage-specific test methods."""
def _corrupt_cache_entry(
@ -135,6 +135,8 @@ class StorageTestMixin:
def test_corrupted_cache_entry_is_a_miss(self, caplog):
with self._middleware() as mw:
spider = mw.crawler.spider
assert spider
assert mw.crawler.stats
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)
@ -155,6 +157,8 @@ class StorageTestMixin:
def test_corrupted_cache_entry_ignore_missing(self):
with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
spider = mw.crawler.spider
assert spider
assert mw.crawler.stats
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)
@ -180,7 +184,7 @@ class StorageTestMixin:
self.assertEqualResponse(response, cached_response)
class PolicyTestMixin:
class PolicyTestMixin(TestBase):
"""Mixin containing policy-specific test methods."""
def test_dont_cache(self):
@ -302,6 +306,7 @@ class DummyPolicyTestMixin(PolicyTestMixin):
assert mw.process_request(self.request) is None
fresh_response = self.response.replace(body=b"new body")
response = mw.process_response(self.request, fresh_response)
assert isinstance(response, Response)
self.assertEqualResponse(self.response, response)
assert "cached" in response.flags
assert mw.stats.get_value("httpcache/revalidate") == 1
@ -313,12 +318,12 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
@staticmethod
def _process_requestresponse(
mw: HttpCacheMiddleware, request: Request, response: Response | None
) -> Response | Request:
result = None
) -> Response:
result: Request | Response | None = None
try:
result = mw.process_request(request)
if result:
assert isinstance(result, (Request, Response))
assert isinstance(result, Response)
return result
assert response is not None
result = mw.process_response(request, response)
@ -346,6 +351,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
res2 = self._process_requestresponse(mw, req0, res0)
assert "cached" not in res2.flags
res3 = mw.process_request(req0)
assert isinstance(res3, Response)
assert "cached" in res3.flags
self.assertEqualResponse(res2, res3)
# request with no-cache directive must not return cached response
@ -634,6 +640,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
assert mw.process_request(req0) is None
res1 = mw.process_exception(req0, e("foo"))
# Use cached response as recovery
assert isinstance(res1, Response)
assert "cached" in res1.flags
self.assertEqualResponse(res0, res1)
# Do not use cached response for unhandled exceptions
@ -684,26 +691,22 @@ class DbmStorageTestMixin(StorageTestMixin):
class TestFilesystemStorageWithDummyPolicy(
TestBase, FilesystemStorageTestMixin, DummyPolicyTestMixin
FilesystemStorageTestMixin, DummyPolicyTestMixin
):
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
class TestFilesystemStorageWithRFC2616Policy(
TestBase, FilesystemStorageTestMixin, RFC2616PolicyTestMixin
FilesystemStorageTestMixin, RFC2616PolicyTestMixin
):
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
class TestDbmStorageWithDummyPolicy(
TestBase, DbmStorageTestMixin, DummyPolicyTestMixin
):
class TestDbmStorageWithDummyPolicy(DbmStorageTestMixin, DummyPolicyTestMixin):
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
class TestDbmStorageWithRFC2616Policy(
TestBase, DbmStorageTestMixin, RFC2616PolicyTestMixin
):
class TestDbmStorageWithRFC2616Policy(DbmStorageTestMixin, RFC2616PolicyTestMixin):
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"

58
tests/test_downloadermiddleware_httpcompression.py
View File

@ -3,6 +3,7 @@ from importlib.util import find_spec
from io import BytesIO
from logging import WARNING
from pathlib import Path
from typing import Any
import pytest
from w3lib.encoding import resolve_encoding
@ -15,6 +16,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWar
from scrapy.http import HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils._compression import _DecompressionMaxSizeExceeded
from scrapy.utils.gz import gunzip
from scrapy.utils.test import get_crawler
from tests import tests_datadir
@ -72,6 +74,7 @@ class TestHttpCompression:
def setup_method(self):
self.crawler = get_crawler(Spider)
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
assert self.crawler.stats
self.crawler.stats.open_spider()
def _getresponse(self, coding: str) -> Response:
@ -96,7 +99,8 @@ class TestHttpCompression:
)
return response
def assertStatsEqual(self, key, value):
def assertStatsEqual(self, key: str, value: Any) -> None:
assert self.crawler.stats
assert self.crawler.stats.get_value(key) == value, str(
self.crawler.stats.get_stats()
)
@ -145,6 +149,7 @@ class TestHttpCompression:
def test_process_response_gzip(self):
response = self._getresponse("gzip")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"gzip"
@ -159,6 +164,7 @@ class TestHttpCompression:
_skip_if_no_br()
response = self._getresponse("br")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"br"
newresponse = self.mw.process_response(request, response)
@ -172,6 +178,7 @@ class TestHttpCompression:
if find_spec("brotli") is not None or find_spec("brotlicffi") is not None:
pytest.skip("Requires not having brotli support")
response = self._getresponse("br")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"br"
caplog.clear()
@ -201,6 +208,7 @@ class TestHttpCompression:
if not check_key.startswith("zstd-"):
continue
response = self._getresponse(check_key)
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"zstd"
newresponse = self.mw.process_response(request, response)
@ -216,6 +224,7 @@ class TestHttpCompression:
if find_spec("zstandard") is not None:
pytest.skip("Requires not having zstandard support")
response = self._getresponse("zstd-static-content-size")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"zstd"
caplog.clear()
@ -239,6 +248,7 @@ class TestHttpCompression:
def test_process_response_rawdeflate(self):
response = self._getresponse("rawdeflate")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"deflate"
@ -251,6 +261,7 @@ class TestHttpCompression:
def test_process_response_zlibdelate(self):
response = self._getresponse("zlibdeflate")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"deflate"
@ -275,6 +286,7 @@ class TestHttpCompression:
def test_multipleencodings(self):
response = self._getresponse("gzip")
response.headers["Content-Encoding"] = ["uuencode", "gzip"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -282,6 +294,7 @@ class TestHttpCompression:
def test_multi_compression_single_header(self):
response = self._getresponse("gzip-deflate")
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -293,6 +306,7 @@ class TestHttpCompression:
) -> None:
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = [b"gzip, foo, deflate"]
assert response.request
request = response.request
caplog.clear()
with caplog.at_level(
@ -315,6 +329,7 @@ class TestHttpCompression:
def test_multi_compression_multiple_header(self):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "deflate"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -324,6 +339,7 @@ class TestHttpCompression:
def test_multi_compression_multiple_header_invalid_compression(self):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -332,6 +348,7 @@ class TestHttpCompression:
def test_multi_compression_single_and_multiple_header(self):
response = self._getresponse("gzip-deflate-gzip")
response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -341,6 +358,7 @@ class TestHttpCompression:
def test_multi_compression_single_and_multiple_header_invalid_compression(self):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo,deflate"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -397,9 +415,7 @@ class TestHttpCompression:
self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
def test_process_response_no_content_type_header(self):
headers = {
"Content-Encoding": "identity",
}
headers = {b"Content-Encoding": b"identity"}
plainbody = (
b"<html><head><title>Some page</title>"
b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">'
@ -414,6 +430,7 @@ class TestHttpCompression:
newresponse = self.mw.process_response(request, response)
assert isinstance(newresponse, respcls)
assert isinstance(newresponse, HtmlResponse)
assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312")
self.assertStatsEqual("httpcompression/response_count", 1)
@ -422,6 +439,7 @@ class TestHttpCompression:
def test_process_response_gzipped_contenttype(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/gzip"
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
@ -434,6 +452,7 @@ class TestHttpCompression:
def test_process_response_gzip_app_octetstream_contenttype(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/octet-stream"
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
@ -446,6 +465,7 @@ class TestHttpCompression:
def test_process_response_gzip_binary_octetstream_contenttype(self):
response = self._getresponse("x-gzip")
response.headers["Content-Type"] = "binary/octet-stream"
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
@ -504,6 +524,7 @@ class TestHttpCompression:
def test_process_response_head_request_no_decode_required(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/gzip"
assert response.request
request = response.request
request.method = "HEAD"
response = response.replace(body=None)
@ -513,7 +534,7 @@ class TestHttpCompression:
self.assertStatsEqual("httpcompression/response_count", None)
self.assertStatsEqual("httpcompression/response_bytes", None)
def _test_compression_bomb_setting(self, compression_id):
def _test_compression_bomb_setting(self, compression_id: str) -> None:
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
@ -521,9 +542,12 @@ class TestHttpCompression:
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
mw.process_response(response.request, response)
assert exc_info.value.__cause__.decompressed_size < 1_100_000
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
def test_compression_bomb_setting_br(self):
_skip_if_no_br()
@ -549,6 +573,7 @@ class TestHttpCompression:
mw.open_spider(spider)
response = self._getresponse("bomb-gzip") # 11_511_612 B
assert response.request
caplog.clear()
with (
caplog.at_level(
@ -565,7 +590,7 @@ class TestHttpCompression:
)
]
def _test_compression_bomb_spider_attr(self, compression_id):
def _test_compression_bomb_spider_attr(self, compression_id: str) -> None:
class DownloadMaxSizeSpider(Spider):
download_maxsize = 1_000_000
@ -575,9 +600,12 @@ class TestHttpCompression:
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
mw.process_response(response.request, response)
assert exc_info.value.__cause__.decompressed_size < 1_100_000
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr_br(self):
@ -599,7 +627,7 @@ class TestHttpCompression:
self._test_compression_bomb_spider_attr("zstd")
def _test_compression_bomb_request_meta(self, compression_id):
def _test_compression_bomb_request_meta(self, compression_id: str) -> None:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
@ -607,9 +635,12 @@ class TestHttpCompression:
response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_maxsize"] = 1_000_000
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
mw.process_response(response.request, response)
assert exc_info.value.__cause__.decompressed_size < 1_100_000
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
def test_compression_bomb_request_meta_br(self):
_skip_if_no_br()
@ -789,7 +820,7 @@ class TestHttpCompression:
self._test_download_warnsize_request_meta(caplog, "zstd")
def _get_truncated_response(self, compression_id):
def _get_truncated_response(self, compression_id: str) -> Response:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
@ -797,7 +828,10 @@ class TestHttpCompression:
response = self._getresponse(compression_id)
truncated_body = response.body[: len(response.body) // 2]
response = response.replace(body=truncated_body)
return mw.process_response(response.request, response)
assert response.request
new_response = mw.process_response(response.request, response)
assert isinstance(new_response, Response)
return new_response
def test_process_truncated_response_br(self):
_skip_if_no_br()

6
tests/test_downloadermiddleware_httpproxy.py
View File

@ -14,7 +14,8 @@ class TestHttpProxyMiddleware:
self._oldenv = os.environ.copy()
def teardown_method(self):
os.environ = self._oldenv
os.environ.clear()
os.environ.update(self._oldenv)
def test_not_enabled(self):
crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False})
@ -22,7 +23,8 @@ class TestHttpProxyMiddleware:
HttpProxyMiddleware.from_crawler(crawler)
def test_no_environment_proxies(self):
os.environ = {"dummy_proxy": "reset_env_and_do_not_raise"}
os.environ.clear()
os.environ["dummy_proxy"] = "reset_env_and_do_not_raise"
mw = HttpProxyMiddleware()
for url in ("http://e.com", "https://e.com", "file:///tmp/a"):

20
tests/test_downloadermiddleware_offsite.py
View File

@ -1,4 +1,5 @@
import re
from typing import Any
import pytest
@ -53,7 +54,7 @@ def test_process_request_dont_filter(value, filtered):
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
kwargs = {}
kwargs: dict[str, Any] = {}
if value is not UNSET:
kwargs["dont_filter"] = value
request = Request("https://b.example", **kwargs)
@ -82,7 +83,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
kwargs = {"meta": {}}
kwargs: dict[str, Any] = {"meta": {}}
if allow_offsite is not UNSET:
kwargs["meta"]["allow_offsite"] = allow_offsite
if dont_filter is not UNSET:
@ -105,7 +106,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
)
def test_process_request_no_allowed_domains(value):
crawler = get_crawler(Spider)
kwargs = {}
kwargs: dict[str, Any] = {}
if value is not UNSET:
kwargs["allowed_domains"] = value
crawler.spider = crawler._create_spider(name="a", **kwargs)
@ -152,7 +153,7 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
mw.spider_opened(crawler.spider)
request = Request(url)
if allowed:
assert mw.request_scheduled(request, crawler.spider) is None
mw.request_scheduled(request, crawler.spider)
else:
with pytest.raises(IgnoreRequest):
mw.request_scheduled(request, crawler.spider)
@ -172,7 +173,7 @@ def test_request_scheduled_dont_filter(value, filtered):
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
kwargs = {}
kwargs: dict[str, Any] = {}
if value is not UNSET:
kwargs["dont_filter"] = value
request = Request("https://b.example", **kwargs)
@ -180,7 +181,7 @@ def test_request_scheduled_dont_filter(value, filtered):
with pytest.raises(IgnoreRequest):
mw.request_scheduled(request, crawler.spider)
else:
assert mw.request_scheduled(request, crawler.spider) is None
mw.request_scheduled(request, crawler.spider)
@pytest.mark.parametrize(
@ -193,14 +194,14 @@ def test_request_scheduled_dont_filter(value, filtered):
)
def test_request_scheduled_no_allowed_domains(value):
crawler = get_crawler(Spider)
kwargs = {}
kwargs: dict[str, Any] = {}
if value is not UNSET:
kwargs["allowed_domains"] = value
crawler.spider = crawler._create_spider(name="a", **kwargs)
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
request = Request("https://example.com")
assert mw.request_scheduled(request, crawler.spider) is None
mw.request_scheduled(request, crawler.spider)
def test_request_scheduled_invalid_domains():
@ -210,7 +211,7 @@ def test_request_scheduled_invalid_domains():
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
request = Request("https://a.example")
assert mw.request_scheduled(request, crawler.spider) is None
mw.request_scheduled(request, crawler.spider)
for letter in ("b", "c"):
request = Request(f"https://{letter}.example")
with pytest.raises(IgnoreRequest):
@ -227,6 +228,7 @@ def test_repeated_offsite_domain():
with pytest.raises(IgnoreRequest):
mw.process_request(req1)
assert "other.org" in mw.domains_seen
assert crawler.stats
assert crawler.stats.get_value("offsite/domains") == 1
assert crawler.stats.get_value("offsite/filtered") == 1
with pytest.raises(IgnoreRequest):

20
tests/test_downloadermiddleware_redirect.py
View File

@ -309,7 +309,7 @@ class TestRedirectMiddleware(TestRedirectBase):
url = "http://www.example.com/301"
url2 = "http://www.example.com/redirected"
def _test_passthrough(req):
def _test_passthrough(req: Request) -> None:
rsp = Response(url, headers={"Location": url2}, status=301, request=req)
r = self.mw.process_response(req, rsp)
assert r is rsp
@ -404,15 +404,17 @@ def test_response_referrer_policy(policy, source_url, target_url, expected_refer
status=301,
headers={"Location": target_url, **extra_headers},
)
source_request = redirect_mw.process_response(source_request, response_redirect)
assert isinstance(source_request, Request)
target_request = redirect_mw.process_response(source_request, response_redirect)
assert isinstance(target_request, Request)
assert source_request.headers.get("Referer") == expected_referrer
assert target_request.headers.get("Referer") == expected_referrer
def test_no_warning_when_referer_middleware_present(caplog):
crawler = get_crawler()
crawler.get_spider_middleware = MagicMock(return_value=MagicMock())
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
return_value=MagicMock()
)
mw = build_from_crawler(RedirectMiddleware, crawler)
caplog.clear()
with caplog.at_level(logging.WARNING):
@ -426,7 +428,9 @@ def test_no_warning_when_referer_middleware_present(caplog):
def test_warning_redirect_middleware(caplog):
crawler = get_crawler()
crawler.get_spider_middleware = MagicMock(return_value=None)
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
return_value=None
)
mw = build_from_crawler(RedirectMiddleware, crawler)
with caplog.at_level(logging.WARNING):
mw._engine_started()
@ -449,7 +453,9 @@ def test_warning_subclass(caplog):
pass
crawler = get_crawler()
crawler.get_spider_middleware = MagicMock(return_value=None)
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
return_value=None
)
mw = build_from_crawler(MyRedirectMiddleware, crawler)
with caplog.at_level(logging.WARNING):
mw._engine_started()

16
tests/test_downloadermiddleware_redirect_metarefresh.py
View File

@ -21,7 +21,7 @@ from tests.utils.redirect import (
)
def meta_refresh_body(url, interval=5):
def meta_refresh_body(url: str, interval: int = 5) -> bytes:
html = f"""<html><head><meta http-equiv="refresh" content="{interval};url={url}"/></head></html>"""
return html.encode("utf-8")
@ -34,10 +34,14 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
crawler = get_crawler(Spider)
self.mw = self.mwcls.from_crawler(crawler)
def _body(self, interval=5, url="http://example.org/newpage"):
def _body(
self, interval: int = 5, url: str = "http://example.org/newpage"
) -> bytes:
return meta_refresh_body(url, interval)
def get_response(self, request, location):
def get_response(
self, request: Request, location: str, status: int = 302
) -> Response:
return HtmlResponse(request.url, body=self._body(url=location))
def test_meta_refresh(self):
@ -75,7 +79,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
assert "Content-Length" not in req2.headers, (
"Content-Length header must not be present in redirected request"
)
assert not req2.body, f"Redirected body must be empty, not '{req2.body}'"
assert not req2.body, f"Redirected body must be empty, not {req2.body!r}"
def test_ignore_tags_default(self):
req = Request(url="http://example.org")
@ -142,7 +146,9 @@ def test_meta_refresh_schemes(url, location, target):
def test_warning_meta_refresh_middleware(caplog):
crawler = get_crawler()
crawler.get_spider_middleware = MagicMock(return_value=None)
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
return_value=None
)
mw = build_from_crawler(MetaRefreshMiddleware, crawler)
with caplog.at_level(logging.WARNING):
mw._engine_started()

69
tests/test_downloadermiddleware_retry.py
View File

@ -31,6 +31,7 @@ class TestRetry:
req = Request("http://www.scrapytest.org/503")
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.priority < req.priority
def test_404(self):
@ -53,9 +54,9 @@ class TestRetry:
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
# first retry
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 1
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.meta["retry_times"] == 1
def test_dont_retry_exc(self):
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
@ -68,18 +69,19 @@ class TestRetry:
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
# first retry
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 1
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.meta["retry_times"] == 1
# second retry
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 2
req3 = self.mw.process_response(req2, rsp)
assert isinstance(req3, Request)
assert req3.meta["retry_times"] == 2
# discard it
assert self.mw.process_response(req, rsp) is rsp
assert self.mw.process_response(req3, rsp) is rsp
assert self.crawler.stats
assert self.crawler.stats.get_value("retry/max_reached") == 1
assert (
self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable")
@ -131,6 +133,7 @@ class TestRetry:
self._test_retry_exception(req, exc("foo"))
stats = self.crawler.stats
assert stats
assert stats.get_value("retry/max_reached") == len(exceptions)
assert stats.get_value("retry/count") == len(exceptions) * 2
assert (
@ -149,29 +152,30 @@ class TestRetry:
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
self._test_retry_exception(req, exc("foo"), mw)
def _test_retry_exception(self, req, exception, mw=None):
def _test_retry_exception(
self, req: Request, exception: Exception, mw: RetryMiddleware | None = None
) -> None:
if mw is None:
mw = self.mw
# first retry
req = mw.process_exception(req, exception)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 1
req2 = mw.process_exception(req, exception)
assert isinstance(req2, Request)
assert req2.meta["retry_times"] == 1
# second retry
req = mw.process_exception(req, exception)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 2
req3 = mw.process_exception(req2, exception)
assert isinstance(req3, Request)
assert req3.meta["retry_times"] == 2
# discard it
req = mw.process_exception(req, exception)
assert req is None
assert mw.process_exception(req3, exception) is None
class TestMaxRetryTimes:
invalid_url = "http://www.scrapytest.org/invalid_url"
def get_middleware(self, settings=None):
def get_middleware(self, settings: dict[str, Any] | None = None) -> RetryMiddleware:
crawler = get_crawler(DefaultSpider, settings or {})
crawler.spider = crawler._create_spider()
return RetryMiddleware.from_crawler(crawler)
@ -275,20 +279,18 @@ class TestMaxRetryTimes:
def _test_retry(
self,
req,
exception,
max_retry_times,
middleware=None,
):
middleware = middleware or self.mw
req: Request,
exception: Exception,
max_retry_times: int,
middleware: RetryMiddleware,
) -> None:
for _ in range(max_retry_times):
req = middleware.process_exception(req, exception)
assert isinstance(req, Request)
result = middleware.process_exception(req, exception)
assert isinstance(result, Request)
req = result
# discard it
req = middleware.process_exception(req, exception)
assert req is None
assert middleware.process_exception(req, exception) is None
class TestGetRetryRequest:
@ -428,7 +430,7 @@ class TestGetRetryRequest:
def test_no_spider(self):
request = Request("https://example.com")
with pytest.raises(TypeError):
get_retry_request(request) # pylint: disable=missing-kwoa
get_retry_request(request) # type: ignore[call-arg] # pylint: disable=missing-kwoa
def test_max_retry_times_setting(self):
max_retry_times = 0
@ -471,6 +473,7 @@ class TestGetRetryRequest:
request,
spider=spider,
)
assert new_request
assert new_request.priority == priority_adjust
def test_priority_adjust_argument(self):
@ -482,6 +485,7 @@ class TestGetRetryRequest:
spider=spider,
priority_adjust=priority_adjust,
)
assert new_request
assert new_request.priority == priority_adjust
def test_log_extra_retry_success(self, caplog: pytest.LogCaptureFixture) -> None:
@ -732,6 +736,7 @@ class TestGetRetryRequest:
reason=expected_reason,
stats_base_key=stats_key,
)
assert spider.crawler.stats
for stat in (
f"{stats_key}/count",
f"{stats_key}/reason_count/{expected_reason}",

49
tests/test_downloadermiddleware_robotstxt.py
View File

@ -1,13 +1,13 @@
from __future__ import annotations
import asyncio
from typing import TYPE_CHECKING
from unittest import mock
import pytest
from twisted.internet.defer import Deferred, DeferredList
from twisted.python import failure
from scrapy import signals
from scrapy.downloadermiddlewares.robotstxt import RobotsTxtMiddleware
from scrapy.exceptions import CannotResolveHostError, IgnoreRequest, NotConfigured
from scrapy.http import Request, Response, TextResponse
@ -18,15 +18,13 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from tests.utils.decorators import coroutine_test
from tests.utils.robotstxt import rerp_available
if TYPE_CHECKING:
from scrapy.crawler import Crawler
class TestRobotsTxtMiddleware:
def setup_method(self) -> None:
self.crawler: mock.MagicMock = mock.MagicMock()
self.crawler.settings = Settings()
self.crawler.engine.download_async = mock.AsyncMock()
self.crawler.signals.send_catch_log_async = mock.AsyncMock(return_value=[])
def teardown_method(self):
del self.crawler
@ -37,7 +35,7 @@ class TestRobotsTxtMiddleware:
with pytest.raises(NotConfigured):
RobotsTxtMiddleware(self.crawler)
def _get_successful_crawler(self) -> Crawler:
def _get_successful_crawler(self) -> mock.MagicMock:
crawler = self.crawler
crawler.settings.set("ROBOTSTXT_OBEY", True)
ROBOTS = """
@ -52,8 +50,8 @@ Disallow: /some/randome/page.html
""".encode()
response = TextResponse("http://site.local/robots.txt", body=ROBOTS)
async def return_response(request):
deferred = Deferred()
async def return_response(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.callback, response)
return await maybe_deferred_to_future(deferred)
@ -74,6 +72,21 @@ Disallow: /some/randome/page.html
Request("http://site.local/wiki/Käyttäjä:"), middleware
)
@coroutine_test
async def test_robotstxt_emits_robots_parsed_signal(self):
crawler = self._get_successful_crawler()
middleware = RobotsTxtMiddleware(crawler)
request = Request("http://site.local/allowed")
await self.assertNotIgnored(request, middleware)
calls = [
kwargs
for _, kwargs in crawler.signals.send_catch_log_async.call_args_list
if kwargs.get("signal") is signals.robots_parsed
]
assert len(calls) == 1
assert calls[0]["request"] is request
assert calls[0]["robotparser"] is not None
@coroutine_test
async def test_robotstxt_multiple_reqs(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
@ -113,15 +126,15 @@ Disallow: /some/randome/page.html
Request("http://site.local/static/", meta=meta), middleware
)
def _get_garbage_crawler(self) -> Crawler:
def _get_garbage_crawler(self) -> mock.MagicMock:
crawler = self.crawler
crawler.settings.set("ROBOTSTXT_OBEY", True)
response = Response(
"http://site.local/robots.txt", body=b"GIF89a\xd3\x00\xfe\x00\xa2"
)
async def return_response(request):
deferred = Deferred()
async def return_response(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.callback, response)
return await maybe_deferred_to_future(deferred)
@ -137,13 +150,13 @@ Disallow: /some/randome/page.html
await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)
await self.assertNotIgnored(Request("http://site.local/static/"), middleware)
def _get_emptybody_crawler(self) -> Crawler:
def _get_emptybody_crawler(self) -> mock.MagicMock:
crawler = self.crawler
crawler.settings.set("ROBOTSTXT_OBEY", True)
response = Response("http://site.local/robots.txt")
async def return_response(request):
deferred = Deferred()
async def return_response(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.callback, response)
return await maybe_deferred_to_future(deferred)
@ -163,8 +176,8 @@ Disallow: /some/randome/page.html
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
err = CannotResolveHostError("Robotstxt address not found")
async def return_failure(request):
deferred = Deferred()
async def return_failure(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.errback, failure.Failure(err))
return await maybe_deferred_to_future(deferred)
@ -191,8 +204,8 @@ Disallow: /some/randome/page.html
async def test_ignore_robotstxt_request(self):
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
async def ignore_request(request):
deferred = Deferred()
async def ignore_request(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.errback, failure.Failure(IgnoreRequest()))
return await maybe_deferred_to_future(deferred)
@ -219,7 +232,7 @@ Disallow: /some/randome/page.html
@coroutine_test
async def test_robotstxt_local_file(self):
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
middleware.process_request_2 = mock.MagicMock()
middleware.process_request_2 = mock.MagicMock() # type: ignore[method-assign]
await middleware.process_request(Request("data:text/plain,Hello World data"))
assert not middleware.process_request_2.called

12
tests/test_engine_download.py
View File

@ -116,6 +116,18 @@ class TestEngineDownloadAsync:
engine._slot.add_request.assert_called_once_with(request)
engine._slot.remove_request.assert_called_once_with(request)
@coroutine_test
async def test_download_async_fetch_needs_spider(self, engine):
engine._downloader_fetch_needs_spider = True
request = Request("http://example.com")
response = Response("http://example.com", body=b"test body")
engine.spider = Mock()
engine.downloader.fetch.return_value = defer.succeed(response)
result = await self._download(engine, request)
assert result == response
engine.downloader.fetch.assert_called_once_with(request, engine.spider)
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestEngineDownload(TestEngineDownloadAsync):

11
tests/test_engine_loop.py
View File

@ -6,10 +6,9 @@ from typing import TYPE_CHECKING, Any
from scrapy import Request, Spider, signals
from scrapy.core.scheduler import BaseScheduler
from scrapy.utils.asyncio import call_later
from scrapy.utils.asyncio import call_later, sleep
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils import async_sleep
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
@ -65,23 +64,23 @@ class TestMain:
async def start(self):
yield Request("data:,a")
await async_sleep(seconds)
await sleep(seconds)
self.crawler.engine._slot.scheduler.pause()
self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))
# During this time, the scheduler reports having requests but
# returns None.
await async_sleep(seconds)
await sleep(seconds)
self.crawler.engine._slot.scheduler.unpause()
# The scheduler request is processed.
await async_sleep(seconds)
await sleep(seconds)
yield Request("data:,c")
await async_sleep(seconds)
await sleep(seconds)
self.crawler.engine._slot.scheduler.pause()
self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d"))

106
tests/test_extension_throttle.py
View File

@ -3,7 +3,7 @@ from unittest.mock import Mock
import pytest
from scrapy import Request, Spider
from scrapy import Request
from scrapy.exceptions import NotConfigured
from scrapy.extensions.throttle import AutoThrottle
from scrapy.http.response import Response
@ -25,6 +25,13 @@ def get_crawler(settings=None, spidercls=None):
return _get_crawler(settings_dict=settings, spidercls=spidercls)
def _mock_downloader(crawler):
"""Give *crawler* a mock engine, whose downloader AutoThrottle reads."""
crawler.engine = Mock()
crawler.engine.downloader.slots = {}
return crawler.engine.downloader
@pytest.mark.parametrize(
("value", "expected"),
[
@ -60,29 +67,21 @@ def test_target_concurrency_invalid(value):
@pytest.mark.parametrize(
("spider", "setting", "expected"),
("setting", "expected"),
[
(UNSET, UNSET, DOWNLOAD_DELAY),
(1.0, UNSET, 1.0),
(UNSET, 1.0, 1.0),
(1.0, 2.0, 1.0),
(3.0, 2.0, 3.0),
(UNSET, DOWNLOAD_DELAY),
(1.0, 1.0),
],
)
def test_mindelay_definition(spider, setting, expected):
def test_mindelay_definition(setting, expected):
settings = {}
if setting is not UNSET:
settings["DOWNLOAD_DELAY"] = setting
class _TestSpider(Spider):
name = "test"
if spider is not UNSET:
_TestSpider.download_delay = spider
crawler = get_crawler(settings, _TestSpider)
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
at._spider_opened(_TestSpider())
_mock_downloader(crawler)
at._spider_opened(DefaultSpider())
assert at.mindelay == expected
@ -99,58 +98,43 @@ def test_maxdelay_definition(value, expected):
settings["AUTOTHROTTLE_MAX_DELAY"] = value
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
_mock_downloader(crawler)
at._spider_opened(DefaultSpider())
assert at.maxdelay == expected
@pytest.mark.parametrize(
("min_spider", "min_setting", "start_setting", "expected"),
("min_setting", "start_setting", "expected"),
[
(UNSET, UNSET, UNSET, AUTOTHROTTLE_START_DELAY),
(AUTOTHROTTLE_START_DELAY - 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY),
(AUTOTHROTTLE_START_DELAY + 1.0, UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0),
(UNSET, AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY),
(UNSET, AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0),
(UNSET, UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0),
(UNSET, UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0),
(
AUTOTHROTTLE_START_DELAY + 1.0,
AUTOTHROTTLE_START_DELAY + 2.0,
UNSET,
AUTOTHROTTLE_START_DELAY + 1.0,
),
(UNSET, UNSET, AUTOTHROTTLE_START_DELAY),
(AUTOTHROTTLE_START_DELAY - 1.0, UNSET, AUTOTHROTTLE_START_DELAY),
(AUTOTHROTTLE_START_DELAY + 1.0, UNSET, AUTOTHROTTLE_START_DELAY + 1.0),
(UNSET, AUTOTHROTTLE_START_DELAY - 1.0, AUTOTHROTTLE_START_DELAY - 1.0),
(UNSET, AUTOTHROTTLE_START_DELAY + 1.0, AUTOTHROTTLE_START_DELAY + 1.0),
(
AUTOTHROTTLE_START_DELAY + 2.0,
UNSET,
AUTOTHROTTLE_START_DELAY + 1.0,
AUTOTHROTTLE_START_DELAY + 2.0,
),
(
AUTOTHROTTLE_START_DELAY + 1.0,
UNSET,
AUTOTHROTTLE_START_DELAY + 2.0,
AUTOTHROTTLE_START_DELAY + 2.0,
),
],
)
def test_startdelay_definition(min_spider, min_setting, start_setting, expected):
def test_startdelay_definition(min_setting, start_setting, expected):
settings = {}
if min_setting is not UNSET:
settings["DOWNLOAD_DELAY"] = min_setting
if start_setting is not UNSET:
settings["AUTOTHROTTLE_START_DELAY"] = start_setting
class _TestSpider(Spider):
name = "test"
if min_spider is not UNSET:
_TestSpider.download_delay = min_spider
crawler = get_crawler(settings, _TestSpider)
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
spider = _TestSpider()
at._spider_opened(spider)
assert spider.download_delay == expected
downloader = _mock_downloader(crawler)
at._spider_opened(DefaultSpider())
assert downloader._delay == expected
@pytest.mark.parametrize(
@ -174,15 +158,13 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected)
def test_skipped(meta, slot):
crawler = get_crawler()
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
at._spider_opened(spider)
request = Request("https://example.com", meta=meta)
crawler.engine = Mock()
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
if slot is not None:
crawler.engine.downloader.slots[slot] = object()
downloader.slots[slot] = object()
at._adjust_delay = None # Raise exception if called.
at._response_downloaded(None, request, spider)
@ -204,18 +186,16 @@ def test_adjustment(download_latency, target_concurrency, slot_delay, expected):
settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency}
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
response = Response(request.url)
crawler.engine = Mock()
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
slot = Mock()
slot.delay = slot_delay
crawler.engine.downloader.slots["foo"] = slot
downloader.slots["foo"] = slot
at._response_downloaded(response, request, spider)
@ -240,18 +220,16 @@ def test_adjustment_limits(mindelay, maxdelay, expected):
}
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
response = Response(request.url)
crawler.engine = Mock()
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
slot = Mock()
slot.delay = slot_delay
crawler.engine.downloader.slots["foo"] = slot
downloader.slots["foo"] = slot
at._response_downloaded(response, request, spider)
@ -272,18 +250,16 @@ def test_adjustment_bad_response(
settings = {"AUTOTHROTTLE_TARGET_CONCURRENCY": target_concurrency}
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": download_latency, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
response = Response(request.url, status=400)
crawler.engine = Mock()
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
slot = Mock()
slot.delay = slot_delay
crawler.engine.downloader.slots["foo"] = slot
downloader.slots["foo"] = slot
at._response_downloaded(response, request, spider)
@ -294,19 +270,17 @@ def test_debug(caplog):
settings = {"AUTOTHROTTLE_DEBUG": True}
crawler = get_crawler(settings)
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": 1.0, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
response = Response(request.url, body=b"foo")
crawler.engine = Mock()
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
slot = Mock()
slot.delay = 2.0
slot.transferring = (None, None)
crawler.engine.downloader.slots["foo"] = slot
downloader.slots["foo"] = slot
caplog.clear()
with caplog.at_level(INFO):
@ -324,19 +298,17 @@ def test_debug(caplog):
def test_debug_disabled(caplog):
crawler = get_crawler()
at = build_from_crawler(AutoThrottle, crawler)
downloader = _mock_downloader(crawler)
spider = DefaultSpider()
at._spider_opened(spider)
meta = {"download_latency": 1.0, "download_slot": "foo"}
request = Request("https://example.com", meta=meta)
response = Response(request.url, body=b"foo")
crawler.engine = Mock()
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
slot = Mock()
slot.delay = 2.0
slot.transferring = (None, None)
crawler.engine.downloader.slots["foo"] = slot
downloader.slots["foo"] = slot
caplog.clear()
with caplog.at_level(INFO):

22
tests/test_http2_client_protocol.py
View File

@ -30,7 +30,7 @@ from scrapy.utils.defer import (
deferred_from_coro,
maybe_deferred_to_future,
)
from tests.mockserver.http_resources import LeafResource, Status
from tests.mockserver.http_resources import LeafResource, Status, put_child
from tests.mockserver.utils import ssl_context_factory
if TYPE_CHECKING:
@ -199,18 +199,18 @@ class TestHttps2ClientProtocol:
@pytest.fixture
def site(self, tmp_path):
r = File(str(tmp_path))
r.putChild(b"get-data-html-small", GetDataHtmlSmall())
r.putChild(b"get-data-html-large", GetDataHtmlLarge())
put_child(r, b"get-data-html-small", GetDataHtmlSmall())
put_child(r, b"get-data-html-large", GetDataHtmlLarge())
r.putChild(b"post-data-json-small", PostDataJsonSmall())
r.putChild(b"post-data-json-large", PostDataJsonLarge())
put_child(r, b"post-data-json-small", PostDataJsonSmall())
put_child(r, b"post-data-json-large", PostDataJsonLarge())
r.putChild(b"dataloss", Dataloss())
r.putChild(b"no-content-length-header", NoContentLengthHeader())
r.putChild(b"status", Status())
r.putChild(b"query-params", QueryParams())
r.putChild(b"timeout", TimeoutResponse())
r.putChild(b"request-headers", RequestHeaders())
put_child(r, b"dataloss", Dataloss())
put_child(r, b"no-content-length-header", NoContentLengthHeader())
put_child(r, b"status", Status())
put_child(r, b"query-params", QueryParams())
put_child(r, b"timeout", TimeoutResponse())
put_child(r, b"request-headers", RequestHeaders())
return Site(r, timeout=None)
@async_yield_fixture # type: ignore[untyped-decorator]

29
tests/test_http_headers.py
View File

@ -6,9 +6,6 @@ from scrapy.http import Headers
class TestHeaders:
def assertSortedEqual(self, first, second, msg=None):
assert sorted(first) == sorted(second), msg
def test_basics(self):
h = Headers({"Content-Type": "text/html", "Content-Length": 1234})
assert h["Content-Type"]
@ -39,7 +36,7 @@ class TestHeaders:
assert h["X-Forwarded-For"] == b"ip2"
assert h.get("X-Forwarded-For") == b"ip2"
assert h.getlist("X-Forwarded-For") == [b"ip1", b"ip2"]
assert h.getlist("X-Forwarded-For") is not hlist
assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]
def test_multivalue_for_one_header(self):
h = Headers((("a", "b"), ("a", "c")))
@ -49,19 +46,19 @@ class TestHeaders:
def test_encode_utf8(self):
h = Headers({"key": "\xa3"}, encoding="utf-8")
key, val = dict(h).popitem()
key, val = dict(h.items()).popitem()
assert isinstance(key, bytes), key
assert isinstance(val[0], bytes), val[0]
assert val[0] == b"\xc2\xa3"
def test_encode_latin1(self):
h = Headers({"key": "\xa3"}, encoding="latin1")
_, val = dict(h).popitem()
_, val = dict(h.items()).popitem()
assert val[0] == b"\xa3"
def test_encode_multiple(self):
h = Headers({"key": ["\xa3"]}, encoding="utf-8")
_, val = dict(h).popitem()
_, val = dict(h.items()).popitem()
assert val[0] == b"\xc2\xa3"
def test_delete_and_contains(self):
@ -75,7 +72,7 @@ class TestHeaders:
h = Headers()
hlist = ["ip1", "ip2"]
olist = h.setdefault("X-Forwarded-For", hlist)
assert h.getlist("X-Forwarded-For") is not hlist
assert h.getlist("X-Forwarded-For") is not hlist # type: ignore[comparison-overlap]
assert h.getlist("X-Forwarded-For") is olist
h = Headers()
@ -87,16 +84,16 @@ class TestHeaders:
idict = {"Content-Type": "text/html", "X-Forwarded-For": ["ip1", "ip2"]}
h = Headers(idict)
assert dict(h) == {
assert dict(h.items()) == {
b"Content-Type": [b"text/html"],
b"X-Forwarded-For": [b"ip1", b"ip2"],
}
self.assertSortedEqual(h.keys(), [b"X-Forwarded-For", b"Content-Type"])
self.assertSortedEqual(
h.items(),
[(b"X-Forwarded-For", [b"ip1", b"ip2"]), (b"Content-Type", [b"text/html"])],
)
self.assertSortedEqual(h.values(), [b"ip2", b"text/html"])
assert sorted(h.keys()) == [b"Content-Type", b"X-Forwarded-For"]
assert sorted(h.items()) == [
(b"Content-Type", [b"text/html"]),
(b"X-Forwarded-For", [b"ip1", b"ip2"]),
]
assert set(h.values()) == {b"ip2", b"text/html"}
def test_update(self):
h = Headers()
@ -162,4 +159,4 @@ class TestHeaders:
with pytest.raises(TypeError, match="Unsupported value type"):
Headers().setdefault("foo", object())
with pytest.raises(TypeError, match="Unsupported value type"):
Headers().setlist("foo", [object()])
Headers().setlist("foo", [object()]) # type: ignore[list-item]

3
tests/test_http_request.py
View File

@ -1,4 +1,5 @@
import xmlrpc.client
from typing import Any
import pytest
@ -17,7 +18,7 @@ class TestXmlRpcRequest(TestRequestBase):
default_method = "POST"
default_headers = {b"Content-Type": [b"text/xml"]}
def _test_request(self, **kwargs):
def _test_request(self, **kwargs: Any) -> None:
r = self.request_class("http://scrapytest.org/rpc2", **kwargs)
assert r.headers[b"Content-Type"] == b"text/xml"
assert r.body == to_bytes(

75
tests/test_http_request_form.py
View File

@ -2,6 +2,7 @@ from __future__ import annotations
import re
import warnings
from typing import TYPE_CHECKING, Any
from urllib.parse import parse_qs, unquote_to_bytes
import pytest
@ -12,20 +13,32 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
from tests.utils.bases.http_request import TestRequestBase
if TYPE_CHECKING:
from scrapy import Request
def _buildresponse(body, **kwargs):
def _buildresponse(body: bytes | str, **kwargs: Any) -> HtmlResponse:
kwargs.setdefault("body", body)
kwargs.setdefault("url", "http://example.com")
kwargs.setdefault("encoding", "utf-8")
return HtmlResponse(**kwargs)
def _qs(req, encoding="utf-8", to_unicode=False):
qs = req.body if req.method == "POST" else req.url.partition("?")[2]
uqs = unquote_to_bytes(qs)
if to_unicode:
uqs = uqs.decode(encoding)
return parse_qs(uqs, True)
def _query_string(req: Request) -> bytes:
return req.body if req.method == "POST" else req.url.partition("?")[2].encode()
def _qs(req: Request) -> dict[bytes, list[bytes]]:
return parse_qs(unquote_to_bytes(_query_string(req)), True)
def _qs_unicode(req: Request, encoding: str = "utf-8") -> dict[str, list[str]]:
qs = unquote_to_bytes(_query_string(req)).decode(encoding)
return parse_qs(qs, True)
def _assert_query_equal(first: bytes, second: bytes) -> None:
assert sorted(to_unicode(first).split("&")) == sorted(to_unicode(second).split("&"))
# FormRequest.from_response() is deprecated in favor of form2request, so the
@ -34,11 +47,6 @@ def _qs(req, encoding="utf-8", to_unicode=False):
class TestFormRequest(TestRequestBase):
request_class = FormRequest
def assertQueryEqual(self, first, second, msg=None):
first = to_unicode(first).split("&")
second = to_unicode(second).split("&")
assert sorted(first) == sorted(second), msg
def test_init_not_deprecated(self):
# Building a request directly from form data is not deprecated.
with warnings.catch_warnings():
@ -75,20 +83,22 @@ class TestFormRequest(TestRequestBase):
assert fs[b"b"] == [b"2"]
assert fs.get(b"c") is None
data = {"a": "1", "b": "2"}
mapping = {"a": "1", "b": "2"}
fs = _qs(
self.request_class("http://www.example.com/", method="GET", formdata=data)
self.request_class(
"http://www.example.com/", method="GET", formdata=mapping
)
)
assert fs[b"a"] == [b"1"]
assert fs[b"b"] == [b"2"]
def test_default_encoding_bytes(self):
# using default encoding (utf-8)
data = {b"one": b"two", b"price": b"\xc2\xa3 100"}
data: dict[Any, Any] = {b"one": b"two", b"price": b"\xc2\xa3 100"}
r2 = self.request_class("http://www.example.com", formdata=data)
assert r2.method == "POST"
assert r2.encoding == "utf-8"
self.assertQueryEqual(r2.body, b"price=%C2%A3+100&one=two")
_assert_query_equal(r2.body, b"price=%C2%A3+100&one=two")
assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded"
def test_default_encoding_textual_data(self):
@ -97,26 +107,26 @@ class TestFormRequest(TestRequestBase):
r2 = self.request_class("http://www.example.com", formdata=data)
assert r2.method == "POST"
assert r2.encoding == "utf-8"
self.assertQueryEqual(r2.body, b"price=%C2%A3+100&%C2%B5+one=two")
_assert_query_equal(r2.body, b"price=%C2%A3+100&%C2%B5+one=two")
assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded"
def test_default_encoding_mixed_data(self):
# using default encoding (utf-8)
data = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"}
data: dict[Any, Any] = {"\u00b5one": b"two", b"price\xc2\xa3": "\u00a3 100"}
r2 = self.request_class("http://www.example.com", formdata=data)
assert r2.method == "POST"
assert r2.encoding == "utf-8"
self.assertQueryEqual(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100")
_assert_query_equal(r2.body, b"%C2%B5one=two&price%C2%A3=%C2%A3+100")
assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded"
def test_custom_encoding_bytes(self):
data = {b"\xb5 one": b"two", b"price": b"\xa3 100"}
data: dict[Any, Any] = {b"\xb5 one": b"two", b"price": b"\xa3 100"}
r2 = self.request_class(
"http://www.example.com", formdata=data, encoding="latin1"
)
assert r2.method == "POST"
assert r2.encoding == "latin1"
self.assertQueryEqual(r2.body, b"price=%A3+100&%B5+one=two")
_assert_query_equal(r2.body, b"price=%A3+100&%B5+one=two")
assert r2.headers[b"Content-Type"] == b"application/x-www-form-urlencoded"
def test_custom_encoding_textual_data(self):
@ -131,7 +141,7 @@ class TestFormRequest(TestRequestBase):
# using multiples values for a single key
data = {"price": "\xa3 100", "colours": ["red", "blue", "green"]}
r3 = self.request_class("http://www.example.com", formdata=data)
self.assertQueryEqual(
_assert_query_equal(
r3.body, b"colours=red&colours=blue&colours=green&price=%C2%A3+100"
)
@ -173,7 +183,7 @@ class TestFormRequest(TestRequestBase):
assert req.method == "POST"
assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded"
assert req.url == "http://www.example.com/this/post.php"
fs = _qs(req, to_unicode=True)
fs = _qs_unicode(req)
assert set(fs["test £"]) == {"val1", "val2"}
assert set(fs["one"]) == {"two", "three"}
assert fs["test2"] == ["xxx µ"]
@ -196,7 +206,7 @@ class TestFormRequest(TestRequestBase):
assert req.method == "POST"
assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded"
assert req.url == "http://www.example.com/this/post.php"
fs = _qs(req, to_unicode=True, encoding="latin1")
fs = _qs_unicode(req, encoding="latin1")
assert set(fs["test £"]) == {"val1", "val2"}
assert set(fs["one"]) == {"two", "three"}
assert fs["test2"] == ["xxx µ"]
@ -218,7 +228,7 @@ class TestFormRequest(TestRequestBase):
assert req.method == "POST"
assert req.headers[b"Content-type"] == b"application/x-www-form-urlencoded"
assert req.url == "http://www.example.com/this/post.php"
fs = _qs(req, to_unicode=True)
fs = _qs_unicode(req)
assert set(fs["test £"]) == {"val1", "val2"}
assert set(fs["one"]) == {"two", "three"}
assert fs["test2"] == ["xxx µ"]
@ -305,7 +315,10 @@ class TestFormRequest(TestRequestBase):
<input type="hidden" name="two" value="3">
</form>"""
)
req = self.request_class.from_response(response, formdata={"two": None})
req = self.request_class.from_response(
response,
formdata={"two": None}, # type: ignore[arg-type]
)
fs = _qs(req)
assert fs[b"one"] == [b"1"]
assert b"two" not in fs
@ -450,7 +463,7 @@ class TestFormRequest(TestRequestBase):
req = self.request_class.from_response(
response, clickdata={"name": "price in \u00a3"}
)
fs = _qs(req, to_unicode=True)
fs = _qs_unicode(req)
assert fs["price in \u00a3"]
def test_from_response_unicode_clickdata_latin1(self):
@ -466,7 +479,7 @@ class TestFormRequest(TestRequestBase):
req = self.request_class.from_response(
response, clickdata={"name": "price in \u00a5"}
)
fs = _qs(req, to_unicode=True, encoding="latin1")
fs = _qs_unicode(req, encoding="latin1")
assert fs["price in \u00a5"]
def test_from_response_multiple_forms_clickdata(self):
@ -737,7 +750,7 @@ class TestFormRequest(TestRequestBase):
</form>"""
)
req = self.request_class.from_response(res)
fs = _qs(req, to_unicode=True)
fs = _qs_unicode(req)
assert fs == {"i1": ["i1v2"], "i2": ["i2v1"], "i4": ["i4v2", "i4v3"]}
def test_from_response_radio(self):
@ -1022,7 +1035,7 @@ class TestFormRequest(TestRequestBase):
with pytest.raises(
ValueError, match="formdata should be a dict or iterable of tuples"
):
FormRequest.from_response(response, formdata=123)
FormRequest.from_response(response, formdata=123) # type: ignore[arg-type]
def test_form_response_with_custom_invalid_formdata_value_error(self):
"""Test that a ValueError is raised for fault-inducing iterable formdata input"""
@ -1037,7 +1050,7 @@ class TestFormRequest(TestRequestBase):
with pytest.raises(
ValueError, match="formdata should be a dict or iterable of tuples"
):
FormRequest.from_response(response, formdata=("a",))
FormRequest.from_response(response, formdata=("a",)) # type: ignore[arg-type]
def test_get_form_with_xpath_no_form_parent(self):
"""Test that _get_from raised a ValueError when an XPath selects an element

15
tests/test_http_response_text.py
View File

@ -1,6 +1,7 @@
from __future__ import annotations
import codecs
from typing import cast
from unittest import mock
import pytest
@ -14,6 +15,12 @@ from tests.utils.bases.http_response import TestResponseBase
class TestTextResponse(TestResponseBase):
response_class = TextResponse
def _links_response(self) -> TextResponse:
return cast("TextResponse", super()._links_response())
def _links_response_no_href(self) -> TextResponse:
return cast("TextResponse", super()._links_response_no_href())
def test_follow_None_encoding(self):
# unlike the base Response, TextResponse.follow() falls back to the
# response encoding when encoding is None instead of raising
@ -21,7 +28,7 @@ class TestTextResponse(TestResponseBase):
req = r.follow("foo", encoding=None)
assert req.encoding == "cp1252"
def test_replace(self):
def test_replace(self) -> None:
super().test_replace()
r1 = self.response_class(
"http://www.example.com", body="hello", encoding="cp852"
@ -344,7 +351,7 @@ class TestTextResponse(TestResponseBase):
def test_follow_selector_list(self):
resp = self._links_response()
with pytest.raises(ValueError, match="SelectorList"):
resp.follow(resp.css("a"))
resp.follow(resp.css("a")) # type: ignore[arg-type]
def test_follow_selector_invalid(self):
resp = self._links_response()
@ -616,7 +623,7 @@ class CustomResponse(TextResponse):
class TestCustomResponse(TestTextResponse):
response_class = CustomResponse
def test_copy(self):
def test_copy(self) -> None:
super().test_copy()
r1 = self.response_class(
url="https://example.org",
@ -632,7 +639,7 @@ class TestCustomResponse(TestTextResponse):
assert r1.lost == "lost"
assert r2.lost is None
def test_replace(self):
def test_replace(self) -> None:
super().test_replace()
r1 = self.response_class(
url="https://example.org",

5
tests/test_request_attribute_binding.py
View File

@ -85,6 +85,7 @@ class TestCrawl:
url = self.mockserver.url("/status?n=200")
crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
assert isinstance(crawler.spider, SingleRequestSpider)
response = crawler.spider.meta["responses"][0]
assert response.request.url == url
@ -94,6 +95,7 @@ class TestCrawl:
url = self.mockserver.url(f"/status?n={status}")
crawler = get_crawler(SingleRequestSpider)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
assert isinstance(crawler.spider, SingleRequestSpider)
failure = crawler.spider.meta["failure"]
response = failure.value.response
assert failure.request.url == url
@ -111,6 +113,7 @@ class TestCrawl:
},
)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
assert isinstance(crawler.spider, SingleRequestSpider)
failure = crawler.spider.meta["failure"]
assert failure.request.url == url
assert isinstance(failure.value, ZeroDivisionError)
@ -178,6 +181,7 @@ class TestCrawl:
},
)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
assert isinstance(crawler.spider, SingleRequestSpider)
response = crawler.spider.meta["responses"][0]
assert response.body == b"Caught ZeroDivisionError"
assert response.request.url == OVERRIDDEN_URL
@ -201,6 +205,7 @@ class TestCrawl:
},
)
yield crawler.crawl(seed=url, mockserver=self.mockserver)
assert isinstance(crawler.spider, SingleRequestSpider)
response = crawler.spider.meta["responses"][0]
assert response.body == b"Caught ZeroDivisionError"
assert response.request.url == url

26
tests/test_request_cb_kwargs.py
View File

@ -49,6 +49,7 @@ class InjectArgumentsSpiderMiddleware:
async for element in result:
if (
isinstance(element, Request)
and element.callback
and element.callback.__name__ == "parse_spider_mw_2"
):
element.cb_kwargs["from_process_spider_output"] = True
@ -68,7 +69,12 @@ class KeywordArgumentsSpider(MockServerSpider):
checks: list[bool] = []
def _inc_checks(self, count: int = 1) -> None:
assert self.crawler.stats
self.crawler.stats.inc_value("boolean_checks", count)
async def start(self):
assert self.mockserver
data = {"key": "value", "number": 123, "callback": "some_callback"}
yield Request(self.mockserver.url("/first"), self.parse_first, cb_kwargs=data)
yield Request(
@ -89,9 +95,10 @@ class KeywordArgumentsSpider(MockServerSpider):
yield Request(self.mockserver.url("/spider_mw"), self.parse_spider_mw)
def parse_first(self, response, key, number):
assert self.mockserver
self.checks.append(key == "value")
self.checks.append(number == 123)
self.crawler.stats.inc_value("boolean_checks", 2)
self._inc_checks(2)
yield response.follow(
self.mockserver.url("/two"),
self.parse_second,
@ -100,28 +107,28 @@ class KeywordArgumentsSpider(MockServerSpider):
def parse_second(self, response, new_key):
self.checks.append(new_key == "new_value")
self.crawler.stats.inc_value("boolean_checks")
self._inc_checks()
def parse_general(self, response, **kwargs):
if response.url.endswith("/general_with"):
self.checks.append(kwargs["key"] == "value")
self.checks.append(kwargs["number"] == 123)
self.checks.append(kwargs["callback"] == "some_callback")
self.crawler.stats.inc_value("boolean_checks", 3)
self._inc_checks(3)
elif response.url.endswith("/general_without"):
self.checks.append(kwargs == {})
self.crawler.stats.inc_value("boolean_checks")
self._inc_checks()
def parse_no_kwargs(self, response):
self.checks.append(response.url.endswith("/no_kwargs"))
self.crawler.stats.inc_value("boolean_checks")
self._inc_checks()
def parse_default(self, response, key, number=None, default=99):
self.checks.append(response.url.endswith("/default"))
self.checks.append(key == "value")
self.checks.append(number == 123)
self.checks.append(default == 99)
self.crawler.stats.inc_value("boolean_checks", 4)
self._inc_checks(4)
def parse_takes_less(self, response, key, callback):
"""
@ -140,17 +147,18 @@ class KeywordArgumentsSpider(MockServerSpider):
):
self.checks.append(bool(from_process_request))
self.checks.append(bool(from_process_response))
self.crawler.stats.inc_value("boolean_checks", 2)
self._inc_checks(2)
def parse_spider_mw(self, response, from_process_spider_input, from_process_start):
assert self.mockserver
self.checks.append(bool(from_process_spider_input))
self.checks.append(bool(from_process_start))
self.crawler.stats.inc_value("boolean_checks", 2)
self._inc_checks(2)
return Request(self.mockserver.url("/spider_mw_2"), self.parse_spider_mw_2)
def parse_spider_mw_2(self, response, from_process_spider_output):
self.checks.append(bool(from_process_spider_output))
self.crawler.stats.inc_value("boolean_checks", 1)
self._inc_checks()
class TestCallbackKeywordArguments:

58
tests/test_request_dict.py
View File

@ -1,7 +1,10 @@
from typing import Any
import pytest
from twisted.python.failure import Failure
from scrapy import Request, Spider
from scrapy.http import JsonRequest
from scrapy.http import JsonRequest, Response
from scrapy.utils.request import request_from_dict
@ -10,7 +13,7 @@ class CustomRequest(Request):
class TestRequestSerialization:
def setup_method(self):
def setup_method(self) -> None:
self.spider = MethodsSpider()
def test_basic(self):
@ -42,12 +45,14 @@ class TestRequestSerialization:
r = Request("http://www.example.com", body=b"\xc2\xa3")
self._assert_serializes_ok(r)
def _assert_serializes_ok(self, request, spider=None):
def _assert_serializes_ok(
self, request: Request, spider: Spider | None = None
) -> None:
d = request.to_dict(spider=spider)
request2 = request_from_dict(d, spider=spider)
self._assert_same_request(request, request2)
def _assert_same_request(self, r1, r2):
def _assert_same_request(self, r1: Request, r2: Request) -> None:
assert r1.__class__ == r2.__class__
assert r1.url == r2.url
assert r1.callback == r2.callback
@ -64,6 +69,7 @@ class TestRequestSerialization:
assert r1.dont_filter == r2.dont_filter
assert r1.flags == r2.flags
if isinstance(r1, JsonRequest):
assert isinstance(r2, JsonRequest)
assert r1.dumps_kwargs == r2.dumps_kwargs
def test_request_class(self):
@ -83,8 +89,8 @@ class TestRequestSerialization:
def test_reference_callback_serialization(self):
r = Request(
"http://www.example.com",
callback=self.spider.parse_item_reference,
errback=self.spider.handle_error_reference,
callback=self.spider.parse_item_reference, # type: ignore[arg-type,misc]
errback=self.spider.handle_error_reference, # type: ignore[arg-type,misc]
)
self._assert_serializes_ok(r, spider=self.spider)
request_dict = r.to_dict(spider=self.spider)
@ -94,8 +100,8 @@ class TestRequestSerialization:
def test_private_reference_callback_serialization(self):
r = Request(
"http://www.example.com",
callback=self.spider._MethodsSpider__parse_item_reference,
errback=self.spider._MethodsSpider__handle_error_reference,
callback=self.spider._MethodsSpider__parse_item_reference, # type: ignore[attr-defined]
errback=self.spider._MethodsSpider__handle_error_reference, # type: ignore[attr-defined]
)
self._assert_serializes_ok(r, spider=self.spider)
request_dict = r.to_dict(spider=self.spider)
@ -105,7 +111,7 @@ class TestRequestSerialization:
def test_private_callback_serialization(self):
r = Request(
"http://www.example.com",
callback=self.spider._MethodsSpider__parse_item_private,
callback=self.spider._MethodsSpider__parse_item_private, # type: ignore[attr-defined]
errback=self.spider.handle_error,
)
self._assert_serializes_ok(r, spider=self.spider)
@ -113,7 +119,7 @@ class TestRequestSerialization:
def test_mixin_private_callback_serialization(self):
r = Request(
"http://www.example.com",
callback=self.spider._SpiderMixin__mixin_callback,
callback=self.spider._SpiderMixin__mixin_callback, # type: ignore[attr-defined]
errback=self.spider.handle_error,
)
self._assert_serializes_ok(r, spider=self.spider)
@ -127,7 +133,7 @@ class TestRequestSerialization:
self._assert_serializes_ok(r, spider=self.spider)
def test_unserializable_callback1(self):
r = Request("http://www.example.com", callback=lambda x: x)
r = Request("http://www.example.com", callback=lambda x: x) # type: ignore[misc]
with pytest.raises(
ValueError, match="is not an instance method in: <MethodsSpider"
):
@ -144,12 +150,12 @@ class TestRequestSerialization:
class MySpider(Spider):
name = "my_spider"
def parse(self, response):
def parse(self, response: Response) -> None:
pass
spider = MySpider()
r = Request("http://www.example.com", callback=spider.parse)
spider.parse = None
spider.parse = None # type: ignore[method-assign,assignment]
with pytest.raises(ValueError, match="is not an instance method in: <MySpider"):
r.to_dict(spider=spider)
@ -157,7 +163,7 @@ class TestRequestSerialization:
"""Callback method is not available in the spider passed to from_dict"""
spider = SpiderDelegation()
r = Request("http://www.example.com", callback=spider.delegated_callback)
d = r.to_dict(spider=spider)
d = r.to_dict(spider=spider) # type: ignore[arg-type]
with pytest.raises(
ValueError, match="Method 'delegated_callback' not found in: <Spider"
):
@ -165,28 +171,30 @@ class TestRequestSerialization:
class SpiderMixin:
def __mixin_callback(self, response): # pylint: disable=unused-private-member
def __mixin_callback( # pylint: disable=unused-private-member
self, response: Response
) -> None:
pass
class SpiderDelegation:
def delegated_callback(self, response):
def delegated_callback(self, response: Response) -> None:
pass
def parse_item(response):
def parse_item(response: Response) -> None:
pass
def handle_error(failure):
def handle_error(failure: Failure) -> None:
pass
def private_parse_item(response):
def private_parse_item(response: Response) -> None:
pass
def private_handle_error(failure):
def private_handle_error(failure: Failure) -> None:
pass
@ -197,15 +205,17 @@ class MethodsSpider(Spider, SpiderMixin):
__parse_item_reference = private_parse_item
__handle_error_reference = private_handle_error
def __init__(self, **kwargs):
def __init__(self, **kwargs: Any) -> None:
super().__init__(**kwargs)
self.delegated_callback = SpiderDelegation().delegated_callback
def parse_item(self, response):
def parse_item(self, response: Response) -> None:
pass
def handle_error(self, failure):
def handle_error(self, failure: Failure) -> None:
pass
def __parse_item_private(self, response): # pylint: disable=unused-private-member
def __parse_item_private( # pylint: disable=unused-private-member
self, response: Response
) -> None:
pass

43
tests/test_request_left.py
View File

@ -1,57 +1,62 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any
from scrapy.signals import request_left_downloader
from scrapy.spiders import Spider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
from tests.utils.decorators import inline_callbacks_test
if TYPE_CHECKING:
from scrapy import Request
from scrapy.crawler import Crawler
from tests.mockserver.http import MockServer
class SignalCatcherSpider(Spider):
name = "signal_catcher"
def __init__(self, crawler, url, *args, **kwargs):
def __init__(self, crawler: Crawler, url: str, *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
crawler.signals.connect(self.on_request_left, signal=request_left_downloader)
self.caught_times = 0
self.start_urls = [url]
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
def from_crawler(
cls, crawler: Crawler, *args: Any, **kwargs: Any
) -> SignalCatcherSpider:
return cls(crawler, *args, **kwargs)
def on_request_left(self, request, spider):
def on_request_left(self, request: Request, spider: Spider) -> None:
self.caught_times += 1
class TestCatching:
@classmethod
def setup_class(cls):
cls.mockserver = MockServer()
cls.mockserver.__enter__()
@classmethod
def teardown_class(cls):
cls.mockserver.__exit__(None, None, None)
@inline_callbacks_test
def test_success(self):
def test_success(self, mockserver: MockServer):
crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl(self.mockserver.url("/status?n=200"))
yield crawler.crawl(mockserver.url("/status?n=200"))
assert isinstance(crawler.spider, SignalCatcherSpider)
assert crawler.spider.caught_times == 1
@inline_callbacks_test
def test_timeout(self):
def test_timeout(self, mockserver: MockServer):
crawler = get_crawler(SignalCatcherSpider, {"DOWNLOAD_TIMEOUT": 0.1})
yield crawler.crawl(self.mockserver.url("/delay?n=0.2"))
yield crawler.crawl(mockserver.url("/delay?n=0.2"))
assert isinstance(crawler.spider, SignalCatcherSpider)
assert crawler.spider.caught_times == 1
@inline_callbacks_test
def test_disconnect(self):
def test_disconnect(self, mockserver: MockServer):
crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl(self.mockserver.url("/drop"))
yield crawler.crawl(mockserver.url("/drop"))
assert isinstance(crawler.spider, SignalCatcherSpider)
assert crawler.spider.caught_times == 1
@inline_callbacks_test
def test_noconnect(self):
crawler = get_crawler(SignalCatcherSpider)
yield crawler.crawl("http://thereisdefinetelynosuchdomain.com")
assert isinstance(crawler.spider, SignalCatcherSpider)
assert crawler.spider.caught_times == 1

85
tests/test_robotstxt_interface.py
View File

@ -1,30 +1,45 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from scrapy.robotstxt import (
ProtegoRobotParser,
PythonRobotParser,
RerpRobotParser,
RobotParser,
decode_robotstxt,
)
from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
from tests.utils.robotstxt import rerp_available
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
class BaseRobotParserTest:
def _setUp(self, parser_cls):
parser_cls: type[RobotParser]
def _setUp(self, parser_cls: type[RobotParser]) -> None:
self.parser_cls = parser_cls
def _parse(self, robotstxt_body: bytes) -> RobotParser:
# The parser backends only use the crawler to get the spider to log with.
return self.parser_cls.from_crawler(None, robotstxt_body) # type: ignore[arg-type]
def test_allowed(self):
robotstxt_robotstxt_body = (
b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10"
)
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://www.site.local/allowed", "*")
assert not rp.allowed("https://www.site.local/disallowed", "*")
def test_allowed_wildcards(self):
def test_allowed_wildcards(self) -> None:
robotstxt_robotstxt_body = b"""User-agent: first
Disallow: /disallowed/*/end$
@ -32,9 +47,7 @@ class BaseRobotParserTest:
Allow: /*allowed
Disallow: /
"""
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://www.site.local/disallowed", "first")
assert not rp.allowed("https://www.site.local/disallowed/xyz/end", "first")
@ -45,23 +58,19 @@ class BaseRobotParserTest:
assert rp.allowed("https://www.site.local/is_still_allowed", "second")
assert rp.allowed("https://www.site.local/is_allowed_too", "second")
def test_length_based_precedence(self):
def test_length_based_precedence(self) -> None:
robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://www.site.local/page", "*")
def test_order_based_precedence(self):
def test_order_based_precedence(self) -> None:
robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert not rp.allowed("https://www.site.local/page", "*")
def test_empty_response(self):
"""empty response should equal 'allow all'"""
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b"")
rp = self._parse(b"")
assert rp.allowed("https://site.local/", "*")
assert rp.allowed("https://site.local/", "chrome")
assert rp.allowed("https://site.local/index.html", "*")
@ -70,14 +79,22 @@ class BaseRobotParserTest:
def test_garbage_response(self):
"""garbage response should be discarded, equal 'allow all'"""
robotstxt_robotstxt_body = b"GIF89a\xd3\x00\xfe\x00\xa2"
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://site.local/", "*")
assert rp.allowed("https://site.local/", "chrome")
assert rp.allowed("https://site.local/index.html", "*")
assert rp.allowed("https://site.local/disallowed", "*")
def test_crawl_delay(self):
robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n"
rp = self._parse(robotstxt_body)
assert rp.crawl_delay("*") == 10.0
def test_crawl_delay_unset(self):
robotstxt_body = b"User-agent: *\nDisallow: /private\n"
rp = self._parse(robotstxt_body)
assert rp.crawl_delay("*") is None
def test_unicode_url_and_useragent(self):
robotstxt_robotstxt_body = """
User-Agent: *
@ -89,9 +106,7 @@ class BaseRobotParserTest:
User-Agent: UnicödeBöt
Disallow: /some/randome/page.html""".encode()
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://site.local/", "*")
assert not rp.allowed("https://site.local/admin/", "*")
assert not rp.allowed("https://site.local/static/", "*")
@ -102,6 +117,20 @@ class BaseRobotParserTest:
assert not rp.allowed("https://site.local/some/randome/page.html", "UnicödeBöt")
class TestRobotParser:
def test_crawl_delay_unsupported(self):
class AllowAllRobotParser(RobotParser):
@classmethod
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
return cls()
def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
return True
rp = AllowAllRobotParser()
assert rp.crawl_delay("*") is None
class TestDecodeRobotsTxt:
def test_native_string_conversion(self):
robotstxt_body = b"User-agent: *\nDisallow: /\n"
@ -135,21 +164,21 @@ class TestPythonRobotParser(BaseRobotParserTest):
not STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support length based directives precedence.",
)
def test_length_based_precedence(self):
def test_length_based_precedence(self) -> None:
super().test_length_based_precedence()
@pytest.mark.skipif(
STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support order based directives precedence.",
)
def test_order_based_precedence(self):
def test_order_based_precedence(self) -> None:
super().test_order_based_precedence()
@pytest.mark.skipif(
not STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support wildcards.",
)
def test_allowed_wildcards(self):
def test_allowed_wildcards(self) -> None:
super().test_allowed_wildcards()
@ -158,7 +187,7 @@ class TestRerpRobotParser(BaseRobotParserTest):
def setup_method(self):
super()._setUp(RerpRobotParser)
def test_length_based_precedence(self):
def test_length_based_precedence(self) -> None:
pytest.skip("Rerp does not support length based directives precedence.")
@ -166,5 +195,5 @@ class TestProtegoRobotParser(BaseRobotParserTest):
def setup_method(self):
super()._setUp(ProtegoRobotParser)
def test_order_based_precedence(self):
def test_order_based_precedence(self) -> None:
pytest.skip("Protego does not support order based directives precedence.")

11
tests/test_utils_asyncio.py
View File

@ -12,7 +12,9 @@ from scrapy.utils.asyncgen import as_async_generator
from scrapy.utils.asyncio import (
AsyncioLoopingCall,
_parallel_asyncio,
call_later,
is_asyncio_available,
sleep,
)
from tests.utils.decorators import coroutine_test
@ -26,6 +28,15 @@ async def test_is_asyncio_available(reactor_pytest: str) -> None:
assert is_asyncio_available() == (reactor_pytest != "default")
@coroutine_test
async def test_sleep() -> None:
events: list[str] = []
call_later(0.05, events.append, "call_later")
await sleep(0.1)
events.append("sleep")
assert events == ["call_later", "sleep"]
@pytest.mark.only_asyncio
class TestParallelAsyncio:
"""Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync."""

117
tests/test_utils_console.py
View File

@ -1,10 +1,38 @@
from __future__ import annotations
import subprocess
import sys
from importlib.util import find_spec
from io import BytesIO
from typing import TYPE_CHECKING
import pytest
from pexpect import EOF
from scrapy.utils.console import get_shell_embed_func
from scrapy.utils.console import get_shell_embed_func, start_python_console
from scrapy.utils.test import get_testenv
if TYPE_CHECKING:
from pathlib import Path
CONSOLE = """
from scrapy.utils.console import start_python_console
start_python_console(banner="SHELL-READY", shells=["ipython"])
"""
CONSOLE_IN_RUNNING_LOOP = """
import asyncio
from scrapy.utils.console import start_python_console
async def main():
start_python_console(banner="SHELL-READY", shells=["ipython"])
asyncio.run(main())
"""
def test_get_shell_embed_func():
@ -59,3 +87,90 @@ def test_get_shell_embed_func_default():
else:
expected = "_embed_standard_shell"
assert shell.__name__ == expected
@pytest.mark.skipif(find_spec("IPython") is None, reason="IPython is not installed")
class TestIPythonShell:
"""Starting an IPython shell, with and without an asyncio event loop already
running in the calling thread. The latter happens when inspect_response() is
called from a spider callback while using the asyncio reactor."""
@staticmethod
def _env(tmp_path: Path) -> dict[str, str]:
env = get_testenv()
# Keep IPython away from the profile and history of the user running the tests.
env["IPYTHONDIR"] = str(tmp_path)
return env
def test_simple_prompt(self, tmp_path: Path) -> None:
"""IPython falls back to its simple prompt, which needs no event loop,
when stdin is not a TTY."""
env = self._env(tmp_path)
p = subprocess.run(
[sys.executable, "-c", CONSOLE_IN_RUNNING_LOOP],
check=False,
capture_output=True,
encoding="utf-8",
timeout=60,
env=env,
stdin=subprocess.DEVNULL,
)
output = p.stdout + p.stderr
assert "SHELL-READY" in output
assert p.returncode == 0, output
@pytest.mark.skipif(
sys.platform == "win32", reason="requires a POSIX pseudo-terminal"
)
@pytest.mark.parametrize(
"script",
[CONSOLE, CONSOLE_IN_RUNNING_LOOP],
ids=["no_running_loop", "running_loop"],
)
def test_tty(self, tmp_path: Path, script: str) -> None:
"""IPython uses prompt_toolkit, which needs an event loop of its own,
when stdin is a TTY."""
# pexpect only defines spawn, which needs a pseudo-terminal, on POSIX.
from pexpect import spawn # noqa: PLC0415
env = self._env(tmp_path)
env.pop("IPY_TEST_SIMPLE_PROMPT", None)
env["TERM"] = "xterm"
logfile = BytesIO()
p = spawn(
sys.executable,
["-c", script],
env=env,
timeout=60,
)
p.logfile_read = logfile
try:
# Wait for the prompt, which prompt_toolkit draws once it is done
# querying the terminal, before typing into it.
p.expect(r"In \[")
p.sendline("21*2")
p.expect_exact("42")
p.sendline("exit()")
p.expect(EOF)
finally:
p.close()
output = logfile.getvalue().decode()
assert "Traceback" not in output
assert p.exitstatus == 0, output
def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None:
def embed(namespace: dict[str, object], banner: str) -> None:
raise SystemExit
monkeypatch.setattr(
"scrapy.utils.console.get_shell_embed_func", lambda shells: embed
)
start_python_console()
def test_start_python_console_no_shell(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(
"scrapy.utils.console.get_shell_embed_func", lambda shells: None
)
start_python_console()

10
tests/utils/__init__.py
View File

@ -1,6 +1,5 @@
from __future__ import annotations
import asyncio
import os
from pathlib import Path
from typing import TYPE_CHECKING
@ -8,8 +7,6 @@ from typing import TYPE_CHECKING
from twisted.internet.defer import Deferred
from scrapy.settings import Settings, default_settings
from scrapy.utils.asyncio import is_asyncio_available
from scrapy.utils.defer import maybe_deferred_to_future
if TYPE_CHECKING:
from collections.abc import Callable
@ -23,13 +20,6 @@ def twisted_sleep(seconds: float):
return d
async def async_sleep(seconds: float) -> None:
if is_asyncio_available():
await asyncio.sleep(seconds)
else:
await maybe_deferred_to_future(twisted_sleep(seconds))
def get_script_run_env() -> dict[str, str]:
"""Return a OS environment dict suitable to run scripts shipped with tests."""

59
tests/utils/bases/http_request.py
View File

@ -3,8 +3,9 @@ from abc import ABC, abstractmethod
from typing import Any
import pytest
from twisted.python.failure import Failure
from scrapy.http import Headers, Request
from scrapy.http import Headers, Request, Response
from scrapy.http.request import NO_CALLBACK
from scrapy.utils.request import request_to_curl
@ -22,15 +23,15 @@ class TestRequestBase(ABC):
def test_init(self):
# Request requires url in the __init__ method
with pytest.raises(TypeError):
self.request_class()
self.request_class() # type: ignore[call-arg]
# url argument must be basestring
with pytest.raises(TypeError):
self.request_class(123)
self.request_class(123) # type: ignore[arg-type]
# priority argument must be an integer
with pytest.raises(TypeError, match="Request priority not an integer"):
self.request_class("http://www.example.com", priority="1")
self.request_class("http://www.example.com", priority="1") # type: ignore[arg-type]
r = self.request_class("http://www.example.com")
assert isinstance(r.url, str)
@ -205,14 +206,17 @@ class TestRequestBase(ABC):
def test_copy(self):
"""Test Request copy"""
def somecallback():
def somecallback(response: Response) -> None:
pass
def someerrback(failure: Failure) -> None:
pass
r1 = self.request_class(
"http://www.example.com",
flags=["f1", "f2"],
callback=somecallback,
errback=somecallback,
errback=someerrback,
)
r1.meta["foo"] = "bar"
r1.cb_kwargs["key"] = "value"
@ -220,7 +224,7 @@ class TestRequestBase(ABC):
# make sure callbaclks are copied
assert r1.callback is somecallback
assert r1.errback is somecallback
assert r1.errback is someerrback
assert r2.callback is r1.callback
assert r2.errback is r1.errback
@ -251,7 +255,7 @@ class TestRequestBase(ABC):
def test_copy_inherited_classes(self):
"""Test Request children copies preserve their class"""
class CustomRequest(self.request_class):
class CustomRequest(self.request_class): # type: ignore[misc,name-defined]
pass
r1 = CustomRequest("http://www.example.com")
@ -283,7 +287,9 @@ class TestRequestBase(ABC):
assert r4.dont_filter is False
# the cls argument allows changing the resulting class
custom_request_cls = type("CustomRequest", (self.request_class,), {})
custom_request_cls: type[Request] = type(
"CustomRequest", (self.request_class,), {}
)
r5 = r1.replace(cls=custom_request_cls)
assert isinstance(r5, custom_request_cls)
assert r5.url == r1.url
@ -295,33 +301,36 @@ class TestRequestBase(ABC):
def test_immutable_attributes(self):
r = self.request_class("http://example.com")
with pytest.raises(AttributeError):
r.url = "http://example2.com"
r.url = "http://example2.com" # type: ignore[misc]
with pytest.raises(AttributeError):
r.body = "xxx"
r.body = "xxx" # type: ignore[misc,assignment]
def test_callback_and_errback(self):
def a_function():
def a_callback(response: Response) -> None:
pass
def an_errback(failure: Failure) -> None:
pass
r1 = self.request_class("http://example.com")
assert r1.callback is None
assert r1.errback is None
r2 = self.request_class("http://example.com", callback=a_function)
assert r2.callback is a_function
r2 = self.request_class("http://example.com", callback=a_callback)
assert r2.callback is a_callback
assert r2.errback is None
r3 = self.request_class("http://example.com", errback=a_function)
r3 = self.request_class("http://example.com", errback=an_errback)
assert r3.callback is None
assert r3.errback is a_function
assert r3.errback is an_errback
r4 = self.request_class(
url="http://example.com",
callback=a_function,
errback=a_function,
callback=a_callback,
errback=an_errback,
)
assert r4.callback is a_function
assert r4.errback is a_function
assert r4.callback is a_callback
assert r4.errback is an_errback
r5 = self.request_class(
url="http://example.com",
@ -329,18 +338,18 @@ class TestRequestBase(ABC):
errback=NO_CALLBACK,
)
assert r5.callback is NO_CALLBACK
assert r5.errback is NO_CALLBACK
assert r5.errback is NO_CALLBACK # type: ignore[comparison-overlap]
def test_callback_and_errback_type(self):
with pytest.raises(TypeError):
self.request_class("http://example.com", callback="a_function")
self.request_class("http://example.com", callback="a_function") # type: ignore[arg-type]
with pytest.raises(TypeError):
self.request_class("http://example.com", errback="a_function")
self.request_class("http://example.com", errback="a_function") # type: ignore[arg-type]
with pytest.raises(TypeError):
self.request_class(
url="http://example.com",
callback="a_function",
errback="a_function",
callback="a_function", # type: ignore[arg-type]
errback="a_function", # type: ignore[arg-type]
)
def test_setters(self):

48
tests/utils/bases/http_response.py
View File

@ -7,7 +7,7 @@ import pytest
from w3lib.encoding import resolve_encoding
from scrapy.exceptions import NotSupported
from scrapy.http import Headers, Request, Response
from scrapy.http import Headers, Request, Response, TextResponse
from scrapy.link import Link
from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS
from tests import get_testdata
@ -15,6 +15,8 @@ from tests import get_testdata
if TYPE_CHECKING:
from collections.abc import Iterable
from parsel import Selector
class TestResponseBase(ABC):
@property
@ -25,14 +27,14 @@ class TestResponseBase(ABC):
def test_init(self):
# Response requires url in the constructor
with pytest.raises(TypeError):
self.response_class()
self.response_class() # type: ignore[call-arg]
assert isinstance(
self.response_class("http://example.com/"), self.response_class
)
with pytest.raises(TypeError):
self.response_class(b"http://example.com")
self.response_class(b"http://example.com") # type: ignore[arg-type]
with pytest.raises(TypeError):
self.response_class(url="http://example.com", body={})
self.response_class(url="http://example.com", body={}) # type: ignore[arg-type]
# body can be str or None
assert isinstance(
self.response_class("http://example.com/", body=b""),
@ -67,12 +69,12 @@ class TestResponseBase(ABC):
r = self.response_class("http://www.example.com", status=301)
assert r.status == 301
r = self.response_class("http://www.example.com", status="301")
r = self.response_class("http://www.example.com", status="301") # type: ignore[arg-type]
assert r.status == 301
with pytest.raises(ValueError, match=r"invalid literal for int\(\)"):
self.response_class("http://example.com", status="lala200")
self.response_class("http://example.com", status="lala200") # type: ignore[arg-type]
def test_copy(self):
def test_copy(self) -> None:
"""Test Response copy"""
r1 = self.response_class("http://www.example.com", body=b"Some body")
@ -121,7 +123,7 @@ class TestResponseBase(ABC):
def test_copy_inherited_classes(self):
"""Test Response children copies preserve their class"""
class CustomResponse(self.response_class):
class CustomResponse(self.response_class): # type: ignore[misc,name-defined]
pass
r1 = CustomResponse("http://www.example.com")
@ -129,7 +131,7 @@ class TestResponseBase(ABC):
assert isinstance(r2, CustomResponse)
def test_replace(self):
def test_replace(self) -> None:
"""Test Response.replace() method"""
hdrs = Headers({"key": "value"})
r1 = self.response_class("http://www.example.com")
@ -146,7 +148,9 @@ class TestResponseBase(ABC):
assert r4.body == b""
assert not r4.flags
def _assert_response_values(self, response, encoding, body):
def _assert_response_values(
self, response: TextResponse, encoding: str, body: str | bytes
) -> None:
if isinstance(body, str):
body_unicode = body
body_bytes = body.encode(encoding)
@ -160,15 +164,15 @@ class TestResponseBase(ABC):
assert response.body == body_bytes
assert response.text == body_unicode
def _assert_response_encoding(self, response, encoding):
def _assert_response_encoding(self, response: TextResponse, encoding: str) -> None:
assert response.encoding == resolve_encoding(encoding)
def test_immutable_attributes(self):
r = self.response_class("http://example.com")
with pytest.raises(AttributeError):
r.url = "http://example2.com"
r.url = "http://example2.com" # type: ignore[misc]
with pytest.raises(AttributeError):
r.body = "xxx"
r.body = "xxx" # type: ignore[misc,assignment]
def test_setter_mutable_lazy_loading(self):
"""Mutable attributes are set internally to None only until they are
@ -256,7 +260,7 @@ class TestResponseBase(ABC):
def test_follow_None_url(self):
r = self.response_class("http://example.com")
with pytest.raises(ValueError, match="url can't be None"):
r.follow(None)
r.follow(None) # type: ignore[arg-type]
def test_follow_None_encoding(self):
r = self.response_class("http://example.com")
@ -325,20 +329,20 @@ class TestResponseBase(ABC):
r = self.response_class("http://example.com")
if self.response_class == Response:
with pytest.raises(TypeError):
list(r.follow_all(urls=None))
list(r.follow_all(urls=None)) # type: ignore[arg-type]
with pytest.raises(TypeError):
list(r.follow_all(urls=12345))
list(r.follow_all(urls=12345)) # type: ignore[arg-type]
with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
list(r.follow_all(urls=[None])) # type: ignore[list-item]
else:
with pytest.raises(
ValueError, match="Please supply exactly one of the following arguments"
):
list(r.follow_all(urls=None))
list(r.follow_all(urls=None)) # type: ignore[arg-type]
with pytest.raises(TypeError):
list(r.follow_all(urls=12345))
list(r.follow_all(urls=12345)) # type: ignore[arg-type]
with pytest.raises(ValueError, match="url can't be None"):
list(r.follow_all(urls=[None]))
list(r.follow_all(urls=[None])) # type: ignore[list-item]
@pytest.mark.xfail(
not W3LIB_STRIPS_URLS,
@ -384,14 +388,14 @@ class TestResponseBase(ABC):
def _assert_followed_url(
self,
follow_obj: str | Link,
follow_obj: str | Link | Selector,
target_url: str,
response: Response | None = None,
encoding: str | None = None,
) -> None:
if response is None:
response = self._links_response()
req = response.follow(follow_obj)
req = response.follow(follow_obj) # type: ignore[arg-type]
assert req.url == target_url
if encoding is not None:
assert req.encoding == encoding

13
tests/utils/cmdline.py
View File

@ -46,3 +46,16 @@ def write_recording_editor(editor: Path) -> None:
open (its last argument) into the file given as its first argument."""
editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
editor.chmod(0o755)
def write_recording_browser(browser: Path, recorded: Path) -> None:
"""Create an executable browser script that writes the URL it is asked to
open into *recorded*.
``webbrowser`` only passes the URL to the command from the ``BROWSER``
environment variable, hence the hardcoded output path.
"""
browser.write_text(
f'#!/bin/sh\nprintf "%s" "$1" > "{recorded}"\n', encoding="utf-8"
)
browser.chmod(0o755)

28
tox.ini
View File

@ -5,7 +5,8 @@
[tox]
requires =
sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8
sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10
tox-uv
envlist =
pre-commit
pylint
@ -30,6 +31,7 @@ envlist =
botocore
pypy3
pypy3-extra-deps
benchmark
minversion = 1.7.0
[test-requirements]
@ -109,7 +111,8 @@ commands =
pre-commit run {posargs:--all-files}
[testenv:pylint]
basepython = python3
# Some checks are Python-version-dependent, so pin the version used in CI.
basepython = python3.14
deps =
{[testenv:extra-deps]deps}
pylint==4.0.6
@ -189,8 +192,8 @@ deps =
brotlicffi==1.2.0.0; implementation_name == "pypy"
google-cloud-storage==1.29.0
httpx2[http2,socks]==2.0.0
ipython==7.1.0
ptpython==2.0.1
ipython==8.15.0
ptpython==3.0.23
robotexclusionrulesparser==1.6.2
uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"
zstandard==0.16.0; implementation_name != "pypy"
@ -317,3 +320,20 @@ setenv =
{[min]setenv}
commands =
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore
# CPU benchmarks, tracked on CodSpeed.
#
# pytest-twisted is left out on purpose: benchmarked code must be callable
# synchronously, so tests/benchmarks drives the reactor itself.
[testenv:benchmark]
basepython = python3.14
deps =
pytest >= 8.4.1
pytest-codspeed
passenv =
*codspeed*
*ci*
commands =
pytest {posargs:tests/benchmarks} --codspeed --codspeed-mode=simulation