RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into faster-shutdown-2

This commit is contained in:
Adrian Chaves 2026-08-03 13:28:07 +02:00
parent 4a77c20c7b a9f1770306
commit 25f732b5bb
34 changed files with 612 additions and 306 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.github/dependabot.yml vendored Normal file
View File

@ -0,0 +1,12 @@
version: 2
updates:
- package-ecosystem: github-actions
directory: "/"
schedule:
interval: monthly
groups:
github-actions:
patterns:
- "*"
cooldown:
default-days: 7

6
.github/workflows/auto-close-llm-pr.yml vendored
View File

@ -1,5 +1,7 @@
name: Auto-close LLM PRs
on:
# The workflow only reads the pull request body through the API, it never
# checks out or runs pull request code, so pull_request_target is safe here.
on: # zizmor: ignore[dangerous-triggers]
pull_request_target:
types: [opened]
permissions:
@ -11,7 +13,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check PR body and close if LLM-written
uses: actions/github-script@v6
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |

32
.github/workflows/checks.yml vendored
View File

@ -1,4 +1,8 @@
name: Checks
permissions:
contents: read
on:
push:
branches:
@ -14,6 +18,10 @@ jobs:
checks:
runs-on: ubuntu-latest
timeout-minutes: 30
env:
# Make uv use the interpreter that actions/setup-python installed instead
# of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@ -39,22 +47,32 @@ jobs:
TOXENV: twinecheck
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
docs/requirements.txt
pyproject.toml
tox.ini
- name: Run check
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
run: uvx --with tox-uv tox
pre-commit:
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@v6
- uses: pre-commit/action@v3.0.1
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1

24
.github/workflows/codspeed.yml vendored
View File

@ -22,24 +22,38 @@ permissions: {}
jobs:
benchmark:
runs-on: ubuntu-latest
env:
# Make uv use the interpreter that actions/setup-python installed
# instead of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
permissions:
contents: read
id-token: write # OIDC authentication with CodSpeed
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python 3.14
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: '3.14'
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
pyproject.toml
tox.ini
- name: Install dependencies
# tox must stay on PATH for the CodSpeed action to invoke it.
run: |
pip install --upgrade pip
pip install --upgrade tox
uv tool install --with tox-uv tox
tox -n -e benchmark
- name: Run benchmarks
uses: CodSpeedHQ/action@v4
uses: CodSpeedHQ/action@f22792bfac16f3e14eb9fbea76f4a48e9cc22b93 # v4.19.1
with:
mode: simulation
run: tox -e benchmark

35
.github/workflows/publish.yml vendored
View File

@ -1,4 +1,8 @@
name: Publish
permissions:
contents: read
on:
push:
tags:
@ -9,8 +13,28 @@ concurrency:
cancel-in-progress: true
jobs:
build:
name: Build distribution
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.14"
- run: |
python -m pip install --upgrade build
python -m build
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: python-package-distributions
path: dist/
publish:
name: Upload release to PyPI
needs:
- build
runs-on: ubuntu-latest
environment:
name: pypi
@ -18,12 +42,9 @@ jobs:
permissions:
id-token: write
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
python-version: "3.14"
- run: |
python -m pip install --upgrade build
python -m build
name: python-package-distributions
path: dist/
- name: Publish to PyPI
uses: pypa/gh-action-pypi-publish@release/v1
uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2

56
.github/workflows/tests-macos.yml vendored
View File

@ -1,4 +1,8 @@
name: macOS
permissions:
contents: read
on:
push:
branches:
@ -12,45 +16,63 @@ concurrency:
jobs:
tests:
name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})
runs-on: macos-latest
timeout-minutes: 60
env:
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
# Make uv use the interpreter that actions/setup-python installed instead
# of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13"]
env:
- TOXENV: py
include:
- python-version: '3.14'
env:
TOXENV: py
coverage: true
- python-version: '3.14'
env:
TOXENV: no-reactor
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.14"
env:
TOXENV: py
coverage: true
- python-version: "3.14"
env:
TOXENV: no-reactor
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
pyproject.toml
tox.ini
- name: Install mitmproxy
env:
# mitmproxy needs a newer Python than the oldest matrix entries, so let
# uv download one where no system interpreter is new enough.
UV_PYTHON_PREFERENCE: system
run: uv tool install mitmproxy
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
run: uvx --with tox-uv tox
- name: Upload coverage report
if: ${{ matrix.coverage }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
report_type: test_results

45
.github/workflows/tests-ubuntu.yml vendored
View File

@ -1,4 +1,8 @@
name: Ubuntu
permissions:
contents: read
on:
push:
branches:
@ -12,10 +16,14 @@ concurrency:
jobs:
tests:
name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})
runs-on: ubuntu-latest
timeout-minutes: 60
env:
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
# Make uv use the interpreter that actions/setup-python installed instead
# of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@ -44,10 +52,6 @@ jobs:
env:
TOXENV: no-reactor
coverage: true
# pinned due to https://github.com/pypy/pypy/issues/5388
- python-version: pypy3.11-7.3.20
env:
TOXENV: pypy3
# min deps
- python-version: "3.10.19"
@ -58,10 +62,6 @@ jobs:
env:
TOXENV: min-default-reactor
coverage: true
- python-version: "3.10.19"
env:
TOXENV: min-no-reactor
coverage: true
# pinned due to https://github.com/pypy/pypy/issues/5388
- python-version: pypy3.11-7.3.20
env:
@ -93,10 +93,12 @@ jobs:
coverage: true
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
@ -106,21 +108,32 @@ jobs:
sudo apt-get update
sudo apt-get install libxml2-dev libxslt-dev
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
pyproject.toml
tox.ini
- name: Install mitmproxy
run: pipx install mitmproxy
env:
# mitmproxy needs a newer Python than the oldest matrix entries, so let
# uv download one where no system interpreter is new enough.
UV_PYTHON_PREFERENCE: system
# mitmproxy has no PyPy wheels, so run it on CPython regardless of the
# interpreter under test.
run: uv tool install --python cpython mitmproxy
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
run: uvx --with tox-uv tox
- name: Upload coverage report
if: ${{ matrix.coverage }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
report_type: test_results

48
.github/workflows/tests-windows.yml vendored
View File

@ -1,4 +1,8 @@
name: Windows
permissions:
contents: read
on:
push:
branches:
@ -12,10 +16,14 @@ concurrency:
jobs:
tests:
name: tests (${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})
runs-on: windows-latest
timeout-minutes: 60
env:
PYTEST_ADDOPTS: ${{ matrix.coverage && '-n auto' || '-n auto --no-cov' }}
# Make uv use the interpreter that actions/setup-python installed instead
# of downloading one of its own.
UV_PYTHON_PREFERENCE: only-system
strategy:
fail-fast: false
matrix:
@ -23,22 +31,10 @@ jobs:
- python-version: "3.10"
env:
TOXENV: py
- python-version: "3.11"
env:
TOXENV: py
- python-version: "3.12"
env:
TOXENV: py
- python-version: "3.13"
env:
TOXENV: py
- python-version: "3.14"
env:
TOXENV: py
coverage: true
- python-version: "3.14"
env:
TOXENV: default-reactor
- python-version: "3.14"
env:
TOXENV: no-reactor
@ -56,25 +52,39 @@ jobs:
TOXENV: extra-deps
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v6
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ matrix.python-version }}
- name: Set up uv
uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
with:
cache-dependency-glob: |
pyproject.toml
tox.ini
- name: Install mitmproxy
env:
# mitmproxy needs a newer Python than the oldest matrix entries, so let
# uv download one where no system interpreter is new enough.
UV_PYTHON_PREFERENCE: system
run: uv tool install mitmproxy
- name: Run tests
env: ${{ matrix.env }}
run: |
pip install -U tox
tox
run: uvx --with tox-uv tox
- name: Upload coverage report
if: ${{ matrix.coverage }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
- name: Upload test results
if: ${{ !cancelled() }}
uses: codecov/codecov-action@v5
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
with:
report_type: test_results

7
.pre-commit-config.yaml
View File

@ -27,6 +27,11 @@ repos:
hooks:
- id: sphinx-lint
- repo: https://github.com/scrapy/sphinx-scrapy
rev: 0.8.9
rev: 0.8.10
hooks:
- id: sphinx-scrapy
- repo: https://github.com/zizmorcore/zizmor-pre-commit
rev: v1.28.0
hooks:
- id: zizmor
args: [--no-progress, --fix]

2
docs/requirements.in
View File

@ -5,4 +5,4 @@ sphinx
sphinx-notfound-page
sphinx-rtd-theme
sphinx-rtd-dark-mode
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10

2
docs/requirements.txt
View File

@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0
# via
# -r docs/requirements.in
# sphinx-rtd-dark-mode
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@912ed0507405e16ac60a47dd08195a1cd0ced984
sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@fe176adc1a8577601bc3fa39b590ebed71a7e9b8
# via -r docs/requirements.in
sphinx-sitemap==2.9.0
# via sphinx-scrapy

47
docs/topics/components.rst
View File

@ -9,37 +9,22 @@ A Scrapy component is any class whose objects are built using
That includes the classes that you may assign to the following settings:
- :setting:`ADDONS`
- :setting:`TWISTED_DNS_RESOLVER`
- :setting:`DOWNLOAD_HANDLERS`
- :setting:`DOWNLOADER_MIDDLEWARES`
- :setting:`DUPEFILTER_CLASS`
- :setting:`EXTENSIONS`
- :setting:`FEED_EXPORTERS`
- :setting:`FEED_STORAGES`
- :setting:`ITEM_PIPELINES`
- :setting:`SCHEDULER`
- :setting:`SCHEDULER_DISK_QUEUE`
- :setting:`SCHEDULER_MEMORY_QUEUE`
- :setting:`SCHEDULER_PRIORITY_QUEUE`
- :setting:`SCHEDULER_START_DISK_QUEUE`
- :setting:`SCHEDULER_START_MEMORY_QUEUE`
- :setting:`SPIDER_MIDDLEWARES`
- :setting:`ADDONS`
- :setting:`DOWNLOAD_HANDLERS`
- :setting:`DOWNLOADER_MIDDLEWARES`
- :setting:`DUPEFILTER_CLASS`
- :setting:`EXTENSIONS`
- :setting:`FEED_EXPORTERS`
- :setting:`FEED_STORAGES`
- :setting:`ITEM_PIPELINES`
- :setting:`SCHEDULER`
- :setting:`SCHEDULER_DISK_QUEUE`
- :setting:`SCHEDULER_MEMORY_QUEUE`
- :setting:`SCHEDULER_PRIORITY_QUEUE`
- :setting:`SCHEDULER_START_DISK_QUEUE`
- :setting:`SCHEDULER_START_MEMORY_QUEUE`
- :setting:`SPIDER_MIDDLEWARES`
- :setting:`TWISTED_DNS_RESOLVER`
Third-party Scrapy components may also let you define additional Scrapy
components, usually configurable through :ref:`settings <topics-settings>`, to

8
docs/topics/jobs.rst
View File

@ -83,6 +83,14 @@ stopping it cleanly. Forced, sudden or otherwise unclean shutdown can lead to
data corruption in the job directory, which may prevent the spider from
resuming correctly.
Scrapy version changes
----------------------
The contents of a job directory are an implementation detail of the Scrapy
version that wrote them. A job must be resumed with the same Scrapy version
that paused it; after upgrading or downgrading Scrapy, start a new job with a
new job directory.
Cookies expiration
------------------

2
docs/topics/settings.rst
View File

@ -575,7 +575,7 @@ CONCURRENT_REQUESTS
Default: ``16``
The maximum number of concurrent (i.e. simultaneous) requests that will be
performed by the Scrapy downloader.
performed by the Scrapy downloader. Use ``0`` for no limit.
.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN

16
pyproject.toml
View File

@ -69,8 +69,8 @@ brotli = [
gcs = ["google-cloud-storage>=1.29.0"]
httpx = ["httpx2[http2,socks]>=2.0.0"]
images = ["Pillow>=8.3.2"]
ipython = ["ipython>=7.1.0"]
ptpython = ["ptpython>=2.0.1"]
ipython = ["ipython>=8.15.0"]
ptpython = ["ptpython>=3.0.23"]
robotparser = ["robotexclusionrulesparser>=1.6.2"]
s3 = ["boto3>=1.20.0"]
twisted-http2 = ["Twisted[http2]>=21.7.0"]
@ -125,17 +125,6 @@ module = [
"tests.test_contracts",
"tests.test_core_downloader",
"tests.test_downloader_handler_twisted_ftp",
"tests.test_downloadermiddleware_cookies",
"tests.test_downloadermiddleware_httpauth",
"tests.test_downloadermiddleware_httpcache",
"tests.test_downloadermiddleware_httpcompression",
"tests.test_downloadermiddleware_httpproxy",
"tests.test_downloadermiddleware_offsite",
"tests.test_downloadermiddleware_redirect",
"tests.test_downloadermiddleware_redirect_base",
"tests.test_downloadermiddleware_redirect_metarefresh",
"tests.test_downloadermiddleware_retry",
"tests.test_downloadermiddleware_robotstxt",
"tests.test_downloaderslotssettings",
"tests.test_dupefilters",
"tests.test_engine_loop",
@ -167,7 +156,6 @@ module = [
"tests.test_request_cb_kwargs",
"tests.test_request_dict",
"tests.test_request_left",
"tests.test_robotstxt_interface",
"tests.test_scheduler_base",
"tests.test_settings",
"tests.test_spider",

3
scrapy/core/downloader/__init__.py
View File

@ -140,7 +140,8 @@ class Downloader:
self.active.remove(request)
def needs_backout(self) -> bool:
return len(self.active) >= self.total_concurrency
# A total concurrency of 0 means no limit.
return 0 < self.total_concurrency <= len(self.active)
@_warn_spider_arg
def _get_slot(

7
scrapy/core/downloader/handlers/_httpx.py
View File

@ -92,10 +92,11 @@ class HttpxDownloadHandler(_Base):
self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
self._bind_host: str | None = self._get_bind_address_host()
self._limits: httpx.Limits = httpx.Limits(
# hard limit on simultaneous connections
max_connections=self._pool_size_total,
# hard limit on simultaneous connections (None for no limit, which
# is what a CONCURRENT_REQUESTS of 0 means)
max_connections=self._pool_size_total or None,
# total number of idle connections in the pool (extra ones are closed)
max_keepalive_connections=self._pool_size_total,
max_keepalive_connections=self._pool_size_total or None,
)
self._default_client: httpx.AsyncClient = self._make_client()

4
scrapy/http/request/__init__.py
View File

@ -50,7 +50,9 @@ class VerboseCookie(TypedDict):
secure: NotRequired[bool]
CookiesT: TypeAlias = dict[str | bytes, str | bytes] | list[VerboseCookie]
CookiesT: TypeAlias = (
dict[str | bytes, str | bytes | bool | float | int] | list[VerboseCookie]
)
RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")

28
scrapy/utils/console.py
View File

@ -1,8 +1,9 @@
from __future__ import annotations
import asyncio
import code
from collections.abc import Callable
from functools import wraps
from functools import partial, wraps
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
@ -16,16 +17,8 @@ def _embed_ipython_shell(
namespace: dict[str, Any] | None = None, banner: str = ""
) -> EmbedFuncT:
"""Start an IPython Shell"""
try:
from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415
from IPython.terminal.ipapp import load_default_config # noqa: PLC0415
except ImportError:
from IPython.frontend.terminal.embed import ( # type: ignore[import-not-found,no-redef] # noqa: T100,PLC0415
InteractiveShellEmbed,
)
from IPython.frontend.terminal.ipapp import ( # type: ignore[import-not-found,no-redef] # noqa: PLC0415
load_default_config,
)
from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415
from IPython.terminal.ipapp import load_default_config # noqa: PLC0415
@wraps(_embed_ipython_shell)
def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
@ -38,6 +31,19 @@ def _embed_ipython_shell(
shell = InteractiveShellEmbed.instance(
banner1=banner, user_ns=namespace, config=config
)
# If an asyncio event loop is already running in this thread, e.g. when
# inspect_response() is called from a spider callback while using the
# asyncio reactor, prompt_toolkit cannot run its own event loop here, so
# ask it to run the prompt in a separate thread instead. pt_app is None
# when IPython falls back to its simple prompt, which needs no event loop.
# See https://github.com/scrapy/scrapy/issues/5447
if (pt_app := getattr(shell, "pt_app", None)) is not None:
try:
asyncio.get_running_loop()
except RuntimeError:
pass
else:
pt_app.prompt = partial(pt_app.prompt, in_thread=True)
shell()
return wrapper

17
tests/test_core_downloader.py
View File

@ -299,6 +299,23 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
await self._assert_factory_works(server_url, client_context_factory)
@pytest.mark.parametrize(
("concurrency", "active", "expected"),
[
(2, 1, False),
(2, 2, True),
(0, 0, False),
(0, 2, False),
],
)
def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:
crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})
downloader = Downloader(crawler)
downloader.active = {Request(f"https://example.com/{i}") for i in range(active)}
assert downloader.needs_backout() is expected
downloader.close()
@coroutine_test
async def test_fetch_deprecated_spider_arg():
class CustomDownloader(Downloader):

14
tests/test_downloader_handler_httpx.py
View File

@ -15,6 +15,8 @@ from scrapy.core.downloader.handlers._httpx import (
HttpxDownloadHandler,
)
from scrapy.exceptions import DownloadFailedError
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.test import get_crawler
from tests.utils.bases.download_handlers_http import (
TestHttpBase,
TestHttpProxyBase,
@ -161,3 +163,15 @@ class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
@pytest.mark.requires_internet
class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase):
pass
@pytest.mark.parametrize(("concurrency", "expected"), [(16, 16), (0, None)])
@coroutine_test
async def test_pool_limits(concurrency: int, expected: int | None) -> None:
crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})
handler = build_from_crawler(HttpxDownloadHandler, crawler)
try:
assert handler._limits.max_connections == expected
assert handler._limits.max_keepalive_connections == expected
finally:
await handler.close()

66
tests/test_downloadermiddleware_cookies.py
View File

@ -1,5 +1,6 @@
import logging
from collections.abc import Iterable
from typing import Any
import pytest
@ -219,7 +220,7 @@ class TestCookiesMiddleware:
def test_complex_cookies(self):
# merge some cookies into jar
cookies = [
cookies: list[VerboseCookie] = [
{
"name": "C1",
"value": "value1",
@ -483,13 +484,13 @@ class TestCookiesMiddleware:
def _test_cookie_redirect(
self,
source,
target,
source: str | dict[str, Any],
target: str | dict[str, Any],
*,
cookies1,
cookies2,
):
input_cookies = {"a": "b"}
cookies1: bool,
cookies2: bool,
) -> None:
input_cookies: CookiesT = {"a": "b"}
if not isinstance(source, dict):
source = {"url": source}
@ -551,11 +552,11 @@ class TestCookiesMiddleware:
def _test_cookie_header_redirect(
self,
source,
target,
source: str | dict[str, Any],
target: str | dict[str, Any],
*,
cookies2,
):
cookies2: bool,
) -> None:
"""Test the handling of a user-defined Cookie header when building a
redirect follow-up request.
@ -623,14 +624,14 @@ class TestCookiesMiddleware:
def _test_user_set_cookie_domain_followup(
self,
url1,
url2,
domain,
url1: str,
url2: str,
domain: str,
*,
cookies1,
cookies2,
):
input_cookies = [
cookies1: bool,
cookies2: bool,
) -> None:
input_cookies: list[VerboseCookie] = [
{
"name": "a",
"value": "b",
@ -686,16 +687,16 @@ class TestCookiesMiddleware:
def _test_server_set_cookie_domain_followup(
self,
url1,
url2,
domain,
url1: str,
url2: str,
domain: str,
*,
cookies,
):
cookies: bool,
) -> None:
request1 = Request(url1)
self.mw.process_request(request1)
input_cookies = [
input_cookies: list[VerboseCookie] = [
{
"name": "a",
"value": "b",
@ -747,8 +748,14 @@ class TestCookiesMiddleware:
)
def _test_cookie_redirect_scheme_change(
self, secure, from_scheme, to_scheme, cookies1, cookies2, cookies3
):
self,
secure: bool | object,
from_scheme: str,
to_scheme: str,
cookies1: bool,
cookies2: bool,
cookies3: bool,
) -> None:
"""When a redirect causes the URL scheme to change from *from_scheme*
to *to_scheme*, while domain and port remain the same, and given a
cookie on the initial request with its secure attribute set to
@ -756,10 +763,11 @@ class TestCookiesMiddleware:
initial request (*cookies1*), if it should be kept by the redirect
middleware (*cookies2*), and if it should be present on the Cookie
header in the redirected request (*cookie3*)."""
cookie_kwargs = {}
cookie: VerboseCookie = {"name": "a", "value": "b"}
if secure is not UNSET:
cookie_kwargs["secure"] = secure
input_cookies = [{"name": "a", "value": "b", **cookie_kwargs}]
assert isinstance(secure, bool)
cookie["secure"] = secure
input_cookies = [cookie]
request1 = Request(f"{from_scheme}://a.example", cookies=input_cookies)
self.mw.process_request(request1)

8
tests/test_downloadermiddleware_httpauth.py
View File

@ -1,3 +1,5 @@
from typing import Any
import pytest
from w3lib.http import basic_auth_header
@ -10,8 +12,10 @@ from scrapy.utils.test import get_crawler
_DOMAIN_NOT_SET = object()
def make_mw(user="", passwd="", domain=_DOMAIN_NOT_SET):
settings: dict = {
def make_mw(
user: str = "", passwd: str = "", domain: str | object = _DOMAIN_NOT_SET
) -> HttpAuthMiddleware:
settings: dict[str, Any] = {
"HTTPAUTH_USER": user,
"HTTPAUTH_PASS": passwd,
}

31
tests/test_downloadermiddleware_httpcache.py
View File

@ -94,14 +94,14 @@ class TestBase:
finally:
mw.spider_closed(crawler.spider)
def assertEqualResponse(self, response1, response2):
def assertEqualResponse(self, response1: Response, response2: Response) -> None:
assert response1.url == response2.url
assert response1.status == response2.status
assert response1.headers == response2.headers
assert response1.body == response2.body
class StorageTestMixin:
class StorageTestMixin(TestBase):
"""Mixin containing storage-specific test methods."""
def _corrupt_cache_entry(
@ -135,6 +135,8 @@ class StorageTestMixin:
def test_corrupted_cache_entry_is_a_miss(self, caplog):
with self._middleware() as mw:
spider = mw.crawler.spider
assert spider
assert mw.crawler.stats
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)
@ -155,6 +157,8 @@ class StorageTestMixin:
def test_corrupted_cache_entry_ignore_missing(self):
with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
spider = mw.crawler.spider
assert spider
assert mw.crawler.stats
mw.storage.store_response(spider, self.request, self.response)
self._corrupt_cache_entry(mw.storage, spider, self.request)
@ -180,7 +184,7 @@ class StorageTestMixin:
self.assertEqualResponse(response, cached_response)
class PolicyTestMixin:
class PolicyTestMixin(TestBase):
"""Mixin containing policy-specific test methods."""
def test_dont_cache(self):
@ -302,6 +306,7 @@ class DummyPolicyTestMixin(PolicyTestMixin):
assert mw.process_request(self.request) is None
fresh_response = self.response.replace(body=b"new body")
response = mw.process_response(self.request, fresh_response)
assert isinstance(response, Response)
self.assertEqualResponse(self.response, response)
assert "cached" in response.flags
assert mw.stats.get_value("httpcache/revalidate") == 1
@ -313,12 +318,12 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
@staticmethod
def _process_requestresponse(
mw: HttpCacheMiddleware, request: Request, response: Response | None
) -> Response | Request:
result = None
) -> Response:
result: Request | Response | None = None
try:
result = mw.process_request(request)
if result:
assert isinstance(result, (Request, Response))
assert isinstance(result, Response)
return result
assert response is not None
result = mw.process_response(request, response)
@ -346,6 +351,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
res2 = self._process_requestresponse(mw, req0, res0)
assert "cached" not in res2.flags
res3 = mw.process_request(req0)
assert isinstance(res3, Response)
assert "cached" in res3.flags
self.assertEqualResponse(res2, res3)
# request with no-cache directive must not return cached response
@ -634,6 +640,7 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
assert mw.process_request(req0) is None
res1 = mw.process_exception(req0, e("foo"))
# Use cached response as recovery
assert isinstance(res1, Response)
assert "cached" in res1.flags
self.assertEqualResponse(res0, res1)
# Do not use cached response for unhandled exceptions
@ -684,26 +691,22 @@ class DbmStorageTestMixin(StorageTestMixin):
class TestFilesystemStorageWithDummyPolicy(
TestBase, FilesystemStorageTestMixin, DummyPolicyTestMixin
FilesystemStorageTestMixin, DummyPolicyTestMixin
):
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
class TestFilesystemStorageWithRFC2616Policy(
TestBase, FilesystemStorageTestMixin, RFC2616PolicyTestMixin
FilesystemStorageTestMixin, RFC2616PolicyTestMixin
):
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
class TestDbmStorageWithDummyPolicy(
TestBase, DbmStorageTestMixin, DummyPolicyTestMixin
):
class TestDbmStorageWithDummyPolicy(DbmStorageTestMixin, DummyPolicyTestMixin):
policy_class = "scrapy.extensions.httpcache.DummyPolicy"
class TestDbmStorageWithRFC2616Policy(
TestBase, DbmStorageTestMixin, RFC2616PolicyTestMixin
):
class TestDbmStorageWithRFC2616Policy(DbmStorageTestMixin, RFC2616PolicyTestMixin):
policy_class = "scrapy.extensions.httpcache.RFC2616Policy"

58
tests/test_downloadermiddleware_httpcompression.py
View File

@ -3,6 +3,7 @@ from importlib.util import find_spec
from io import BytesIO
from logging import WARNING
from pathlib import Path
from typing import Any
import pytest
from w3lib.encoding import resolve_encoding
@ -15,6 +16,7 @@ from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWar
from scrapy.http import HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
from scrapy.utils._compression import _DecompressionMaxSizeExceeded
from scrapy.utils.gz import gunzip
from scrapy.utils.test import get_crawler
from tests import tests_datadir
@ -72,6 +74,7 @@ class TestHttpCompression:
def setup_method(self):
self.crawler = get_crawler(Spider)
self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
assert self.crawler.stats
self.crawler.stats.open_spider()
def _getresponse(self, coding: str) -> Response:
@ -96,7 +99,8 @@ class TestHttpCompression:
)
return response
def assertStatsEqual(self, key, value):
def assertStatsEqual(self, key: str, value: Any) -> None:
assert self.crawler.stats
assert self.crawler.stats.get_value(key) == value, str(
self.crawler.stats.get_stats()
)
@ -145,6 +149,7 @@ class TestHttpCompression:
def test_process_response_gzip(self):
response = self._getresponse("gzip")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"gzip"
@ -159,6 +164,7 @@ class TestHttpCompression:
_skip_if_no_br()
response = self._getresponse("br")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"br"
newresponse = self.mw.process_response(request, response)
@ -172,6 +178,7 @@ class TestHttpCompression:
if find_spec("brotli") is not None or find_spec("brotlicffi") is not None:
pytest.skip("Requires not having brotli support")
response = self._getresponse("br")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"br"
caplog.clear()
@ -201,6 +208,7 @@ class TestHttpCompression:
if not check_key.startswith("zstd-"):
continue
response = self._getresponse(check_key)
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"zstd"
newresponse = self.mw.process_response(request, response)
@ -216,6 +224,7 @@ class TestHttpCompression:
if find_spec("zstandard") is not None:
pytest.skip("Requires not having zstandard support")
response = self._getresponse("zstd-static-content-size")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"zstd"
caplog.clear()
@ -239,6 +248,7 @@ class TestHttpCompression:
def test_process_response_rawdeflate(self):
response = self._getresponse("rawdeflate")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"deflate"
@ -251,6 +261,7 @@ class TestHttpCompression:
def test_process_response_zlibdelate(self):
response = self._getresponse("zlibdeflate")
assert response.request
request = response.request
assert response.headers["Content-Encoding"] == b"deflate"
@ -275,6 +286,7 @@ class TestHttpCompression:
def test_multipleencodings(self):
response = self._getresponse("gzip")
response.headers["Content-Encoding"] = ["uuencode", "gzip"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -282,6 +294,7 @@ class TestHttpCompression:
def test_multi_compression_single_header(self):
response = self._getresponse("gzip-deflate")
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -293,6 +306,7 @@ class TestHttpCompression:
) -> None:
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = [b"gzip, foo, deflate"]
assert response.request
request = response.request
caplog.clear()
with caplog.at_level(
@ -315,6 +329,7 @@ class TestHttpCompression:
def test_multi_compression_multiple_header(self):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "deflate"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -324,6 +339,7 @@ class TestHttpCompression:
def test_multi_compression_multiple_header_invalid_compression(self):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo", "deflate"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -332,6 +348,7 @@ class TestHttpCompression:
def test_multi_compression_single_and_multiple_header(self):
response = self._getresponse("gzip-deflate-gzip")
response.headers["Content-Encoding"] = ["gzip", "deflate, gzip"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -341,6 +358,7 @@ class TestHttpCompression:
def test_multi_compression_single_and_multiple_header_invalid_compression(self):
response = self._getresponse("gzip-deflate")
response.headers["Content-Encoding"] = ["gzip", "foo,deflate"]
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
assert newresponse is not response
@ -397,9 +415,7 @@ class TestHttpCompression:
self.assertStatsEqual("httpcompression/response_bytes", len(plainbody))
def test_process_response_no_content_type_header(self):
headers = {
"Content-Encoding": "identity",
}
headers = {b"Content-Encoding": b"identity"}
plainbody = (
b"<html><head><title>Some page</title>"
b'<meta http-equiv="Content-Type" content="text/html; charset=gb2312">'
@ -414,6 +430,7 @@ class TestHttpCompression:
newresponse = self.mw.process_response(request, response)
assert isinstance(newresponse, respcls)
assert isinstance(newresponse, HtmlResponse)
assert newresponse.body == plainbody
assert newresponse.encoding == resolve_encoding("gb2312")
self.assertStatsEqual("httpcompression/response_count", 1)
@ -422,6 +439,7 @@ class TestHttpCompression:
def test_process_response_gzipped_contenttype(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/gzip"
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
@ -434,6 +452,7 @@ class TestHttpCompression:
def test_process_response_gzip_app_octetstream_contenttype(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/octet-stream"
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
@ -446,6 +465,7 @@ class TestHttpCompression:
def test_process_response_gzip_binary_octetstream_contenttype(self):
response = self._getresponse("x-gzip")
response.headers["Content-Type"] = "binary/octet-stream"
assert response.request
request = response.request
newresponse = self.mw.process_response(request, response)
@ -504,6 +524,7 @@ class TestHttpCompression:
def test_process_response_head_request_no_decode_required(self):
response = self._getresponse("gzip")
response.headers["Content-Type"] = "application/gzip"
assert response.request
request = response.request
request.method = "HEAD"
response = response.replace(body=None)
@ -513,7 +534,7 @@ class TestHttpCompression:
self.assertStatsEqual("httpcompression/response_count", None)
self.assertStatsEqual("httpcompression/response_bytes", None)
def _test_compression_bomb_setting(self, compression_id):
def _test_compression_bomb_setting(self, compression_id: str) -> None:
settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
crawler = get_crawler(Spider, settings_dict=settings)
spider = crawler._create_spider("scrapytest.org")
@ -521,9 +542,12 @@ class TestHttpCompression:
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}") # 11_511_612 B
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
mw.process_response(response.request, response)
assert exc_info.value.__cause__.decompressed_size < 1_100_000
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
def test_compression_bomb_setting_br(self):
_skip_if_no_br()
@ -549,6 +573,7 @@ class TestHttpCompression:
mw.open_spider(spider)
response = self._getresponse("bomb-gzip") # 11_511_612 B
assert response.request
caplog.clear()
with (
caplog.at_level(
@ -565,7 +590,7 @@ class TestHttpCompression:
)
]
def _test_compression_bomb_spider_attr(self, compression_id):
def _test_compression_bomb_spider_attr(self, compression_id: str) -> None:
class DownloadMaxSizeSpider(Spider):
download_maxsize = 1_000_000
@ -575,9 +600,12 @@ class TestHttpCompression:
mw.open_spider(spider)
response = self._getresponse(f"bomb-{compression_id}")
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
mw.process_response(response.request, response)
assert exc_info.value.__cause__.decompressed_size < 1_100_000
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
def test_compression_bomb_spider_attr_br(self):
@ -599,7 +627,7 @@ class TestHttpCompression:
self._test_compression_bomb_spider_attr("zstd")
def _test_compression_bomb_request_meta(self, compression_id):
def _test_compression_bomb_request_meta(self, compression_id: str) -> None:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
@ -607,9 +635,12 @@ class TestHttpCompression:
response = self._getresponse(f"bomb-{compression_id}")
response.meta["download_maxsize"] = 1_000_000
assert response.request
with pytest.raises(IgnoreRequest) as exc_info:
mw.process_response(response.request, response)
assert exc_info.value.__cause__.decompressed_size < 1_100_000
cause = exc_info.value.__cause__
assert isinstance(cause, _DecompressionMaxSizeExceeded)
assert cause.decompressed_size < 1_100_000
def test_compression_bomb_request_meta_br(self):
_skip_if_no_br()
@ -789,7 +820,7 @@ class TestHttpCompression:
self._test_download_warnsize_request_meta(caplog, "zstd")
def _get_truncated_response(self, compression_id):
def _get_truncated_response(self, compression_id: str) -> Response:
crawler = get_crawler(Spider)
spider = crawler._create_spider("scrapytest.org")
mw = HttpCompressionMiddleware.from_crawler(crawler)
@ -797,7 +828,10 @@ class TestHttpCompression:
response = self._getresponse(compression_id)
truncated_body = response.body[: len(response.body) // 2]
response = response.replace(body=truncated_body)
return mw.process_response(response.request, response)
assert response.request
new_response = mw.process_response(response.request, response)
assert isinstance(new_response, Response)
return new_response
def test_process_truncated_response_br(self):
_skip_if_no_br()

6
tests/test_downloadermiddleware_httpproxy.py
View File

@ -14,7 +14,8 @@ class TestHttpProxyMiddleware:
self._oldenv = os.environ.copy()
def teardown_method(self):
os.environ = self._oldenv
os.environ.clear()
os.environ.update(self._oldenv)
def test_not_enabled(self):
crawler = get_crawler(Spider, {"HTTPPROXY_ENABLED": False})
@ -22,7 +23,8 @@ class TestHttpProxyMiddleware:
HttpProxyMiddleware.from_crawler(crawler)
def test_no_environment_proxies(self):
os.environ = {"dummy_proxy": "reset_env_and_do_not_raise"}
os.environ.clear()
os.environ["dummy_proxy"] = "reset_env_and_do_not_raise"
mw = HttpProxyMiddleware()
for url in ("http://e.com", "https://e.com", "file:///tmp/a"):

20
tests/test_downloadermiddleware_offsite.py
View File

@ -1,4 +1,5 @@
import re
from typing import Any
import pytest
@ -53,7 +54,7 @@ def test_process_request_dont_filter(value, filtered):
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
kwargs = {}
kwargs: dict[str, Any] = {}
if value is not UNSET:
kwargs["dont_filter"] = value
request = Request("https://b.example", **kwargs)
@ -82,7 +83,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
kwargs = {"meta": {}}
kwargs: dict[str, Any] = {"meta": {}}
if allow_offsite is not UNSET:
kwargs["meta"]["allow_offsite"] = allow_offsite
if dont_filter is not UNSET:
@ -105,7 +106,7 @@ def test_process_request_allow_offsite(allow_offsite, dont_filter, filtered):
)
def test_process_request_no_allowed_domains(value):
crawler = get_crawler(Spider)
kwargs = {}
kwargs: dict[str, Any] = {}
if value is not UNSET:
kwargs["allowed_domains"] = value
crawler.spider = crawler._create_spider(name="a", **kwargs)
@ -152,7 +153,7 @@ def test_request_scheduled_domain_filtering(allowed_domain, url, allowed):
mw.spider_opened(crawler.spider)
request = Request(url)
if allowed:
assert mw.request_scheduled(request, crawler.spider) is None
mw.request_scheduled(request, crawler.spider)
else:
with pytest.raises(IgnoreRequest):
mw.request_scheduled(request, crawler.spider)
@ -172,7 +173,7 @@ def test_request_scheduled_dont_filter(value, filtered):
crawler.spider = crawler._create_spider(name="a", allowed_domains=["a.example"])
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
kwargs = {}
kwargs: dict[str, Any] = {}
if value is not UNSET:
kwargs["dont_filter"] = value
request = Request("https://b.example", **kwargs)
@ -180,7 +181,7 @@ def test_request_scheduled_dont_filter(value, filtered):
with pytest.raises(IgnoreRequest):
mw.request_scheduled(request, crawler.spider)
else:
assert mw.request_scheduled(request, crawler.spider) is None
mw.request_scheduled(request, crawler.spider)
@pytest.mark.parametrize(
@ -193,14 +194,14 @@ def test_request_scheduled_dont_filter(value, filtered):
)
def test_request_scheduled_no_allowed_domains(value):
crawler = get_crawler(Spider)
kwargs = {}
kwargs: dict[str, Any] = {}
if value is not UNSET:
kwargs["allowed_domains"] = value
crawler.spider = crawler._create_spider(name="a", **kwargs)
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
request = Request("https://example.com")
assert mw.request_scheduled(request, crawler.spider) is None
mw.request_scheduled(request, crawler.spider)
def test_request_scheduled_invalid_domains():
@ -210,7 +211,7 @@ def test_request_scheduled_invalid_domains():
mw = OffsiteMiddleware.from_crawler(crawler)
mw.spider_opened(crawler.spider)
request = Request("https://a.example")
assert mw.request_scheduled(request, crawler.spider) is None
mw.request_scheduled(request, crawler.spider)
for letter in ("b", "c"):
request = Request(f"https://{letter}.example")
with pytest.raises(IgnoreRequest):
@ -227,6 +228,7 @@ def test_repeated_offsite_domain():
with pytest.raises(IgnoreRequest):
mw.process_request(req1)
assert "other.org" in mw.domains_seen
assert crawler.stats
assert crawler.stats.get_value("offsite/domains") == 1
assert crawler.stats.get_value("offsite/filtered") == 1
with pytest.raises(IgnoreRequest):

20
tests/test_downloadermiddleware_redirect.py
View File

@ -309,7 +309,7 @@ class TestRedirectMiddleware(TestRedirectBase):
url = "http://www.example.com/301"
url2 = "http://www.example.com/redirected"
def _test_passthrough(req):
def _test_passthrough(req: Request) -> None:
rsp = Response(url, headers={"Location": url2}, status=301, request=req)
r = self.mw.process_response(req, rsp)
assert r is rsp
@ -404,15 +404,17 @@ def test_response_referrer_policy(policy, source_url, target_url, expected_refer
status=301,
headers={"Location": target_url, **extra_headers},
)
source_request = redirect_mw.process_response(source_request, response_redirect)
assert isinstance(source_request, Request)
target_request = redirect_mw.process_response(source_request, response_redirect)
assert isinstance(target_request, Request)
assert source_request.headers.get("Referer") == expected_referrer
assert target_request.headers.get("Referer") == expected_referrer
def test_no_warning_when_referer_middleware_present(caplog):
crawler = get_crawler()
crawler.get_spider_middleware = MagicMock(return_value=MagicMock())
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
return_value=MagicMock()
)
mw = build_from_crawler(RedirectMiddleware, crawler)
caplog.clear()
with caplog.at_level(logging.WARNING):
@ -426,7 +428,9 @@ def test_no_warning_when_referer_middleware_present(caplog):
def test_warning_redirect_middleware(caplog):
crawler = get_crawler()
crawler.get_spider_middleware = MagicMock(return_value=None)
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
return_value=None
)
mw = build_from_crawler(RedirectMiddleware, crawler)
with caplog.at_level(logging.WARNING):
mw._engine_started()
@ -449,7 +453,9 @@ def test_warning_subclass(caplog):
pass
crawler = get_crawler()
crawler.get_spider_middleware = MagicMock(return_value=None)
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
return_value=None
)
mw = build_from_crawler(MyRedirectMiddleware, crawler)
with caplog.at_level(logging.WARNING):
mw._engine_started()

16
tests/test_downloadermiddleware_redirect_metarefresh.py
View File

@ -21,7 +21,7 @@ from tests.utils.redirect import (
)
def meta_refresh_body(url, interval=5):
def meta_refresh_body(url: str, interval: int = 5) -> bytes:
html = f"""<html><head><meta http-equiv="refresh" content="{interval};url={url}"/></head></html>"""
return html.encode("utf-8")
@ -34,10 +34,14 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
crawler = get_crawler(Spider)
self.mw = self.mwcls.from_crawler(crawler)
def _body(self, interval=5, url="http://example.org/newpage"):
def _body(
self, interval: int = 5, url: str = "http://example.org/newpage"
) -> bytes:
return meta_refresh_body(url, interval)
def get_response(self, request, location):
def get_response(
self, request: Request, location: str, status: int = 302
) -> Response:
return HtmlResponse(request.url, body=self._body(url=location))
def test_meta_refresh(self):
@ -75,7 +79,7 @@ class TestMetaRefreshMiddleware(TestRedirectBase):
assert "Content-Length" not in req2.headers, (
"Content-Length header must not be present in redirected request"
)
assert not req2.body, f"Redirected body must be empty, not '{req2.body}'"
assert not req2.body, f"Redirected body must be empty, not {req2.body!r}"
def test_ignore_tags_default(self):
req = Request(url="http://example.org")
@ -142,7 +146,9 @@ def test_meta_refresh_schemes(url, location, target):
def test_warning_meta_refresh_middleware(caplog):
crawler = get_crawler()
crawler.get_spider_middleware = MagicMock(return_value=None)
crawler.get_spider_middleware = MagicMock( # type: ignore[method-assign]
return_value=None
)
mw = build_from_crawler(MetaRefreshMiddleware, crawler)
with caplog.at_level(logging.WARNING):
mw._engine_started()

69
tests/test_downloadermiddleware_retry.py
View File

@ -31,6 +31,7 @@ class TestRetry:
req = Request("http://www.scrapytest.org/503")
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.priority < req.priority
def test_404(self):
@ -53,9 +54,9 @@ class TestRetry:
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
# first retry
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 1
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.meta["retry_times"] == 1
def test_dont_retry_exc(self):
req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
@ -68,18 +69,19 @@ class TestRetry:
rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
# first retry
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 1
req2 = self.mw.process_response(req, rsp)
assert isinstance(req2, Request)
assert req2.meta["retry_times"] == 1
# second retry
req = self.mw.process_response(req, rsp)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 2
req3 = self.mw.process_response(req2, rsp)
assert isinstance(req3, Request)
assert req3.meta["retry_times"] == 2
# discard it
assert self.mw.process_response(req, rsp) is rsp
assert self.mw.process_response(req3, rsp) is rsp
assert self.crawler.stats
assert self.crawler.stats.get_value("retry/max_reached") == 1
assert (
self.crawler.stats.get_value("retry/reason_count/503 Service Unavailable")
@ -131,6 +133,7 @@ class TestRetry:
self._test_retry_exception(req, exc("foo"))
stats = self.crawler.stats
assert stats
assert stats.get_value("retry/max_reached") == len(exceptions)
assert stats.get_value("retry/count") == len(exceptions) * 2
assert (
@ -149,29 +152,30 @@ class TestRetry:
req = Request(f"http://www.scrapytest.org/{exc.__name__}")
self._test_retry_exception(req, exc("foo"), mw)
def _test_retry_exception(self, req, exception, mw=None):
def _test_retry_exception(
self, req: Request, exception: Exception, mw: RetryMiddleware | None = None
) -> None:
if mw is None:
mw = self.mw
# first retry
req = mw.process_exception(req, exception)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 1
req2 = mw.process_exception(req, exception)
assert isinstance(req2, Request)
assert req2.meta["retry_times"] == 1
# second retry
req = mw.process_exception(req, exception)
assert isinstance(req, Request)
assert req.meta["retry_times"] == 2
req3 = mw.process_exception(req2, exception)
assert isinstance(req3, Request)
assert req3.meta["retry_times"] == 2
# discard it
req = mw.process_exception(req, exception)
assert req is None
assert mw.process_exception(req3, exception) is None
class TestMaxRetryTimes:
invalid_url = "http://www.scrapytest.org/invalid_url"
def get_middleware(self, settings=None):
def get_middleware(self, settings: dict[str, Any] | None = None) -> RetryMiddleware:
crawler = get_crawler(DefaultSpider, settings or {})
crawler.spider = crawler._create_spider()
return RetryMiddleware.from_crawler(crawler)
@ -275,20 +279,18 @@ class TestMaxRetryTimes:
def _test_retry(
self,
req,
exception,
max_retry_times,
middleware=None,
):
middleware = middleware or self.mw
req: Request,
exception: Exception,
max_retry_times: int,
middleware: RetryMiddleware,
) -> None:
for _ in range(max_retry_times):
req = middleware.process_exception(req, exception)
assert isinstance(req, Request)
result = middleware.process_exception(req, exception)
assert isinstance(result, Request)
req = result
# discard it
req = middleware.process_exception(req, exception)
assert req is None
assert middleware.process_exception(req, exception) is None
class TestGetRetryRequest:
@ -428,7 +430,7 @@ class TestGetRetryRequest:
def test_no_spider(self):
request = Request("https://example.com")
with pytest.raises(TypeError):
get_retry_request(request) # pylint: disable=missing-kwoa
get_retry_request(request) # type: ignore[call-arg] # pylint: disable=missing-kwoa
def test_max_retry_times_setting(self):
max_retry_times = 0
@ -471,6 +473,7 @@ class TestGetRetryRequest:
request,
spider=spider,
)
assert new_request
assert new_request.priority == priority_adjust
def test_priority_adjust_argument(self):
@ -482,6 +485,7 @@ class TestGetRetryRequest:
spider=spider,
priority_adjust=priority_adjust,
)
assert new_request
assert new_request.priority == priority_adjust
def test_log_extra_retry_success(self, caplog: pytest.LogCaptureFixture) -> None:
@ -732,6 +736,7 @@ class TestGetRetryRequest:
reason=expected_reason,
stats_base_key=stats_key,
)
assert spider.crawler.stats
for stat in (
f"{stats_key}/count",
f"{stats_key}/reason_count/{expected_reason}",

32
tests/test_downloadermiddleware_robotstxt.py
View File

@ -1,7 +1,6 @@
from __future__ import annotations
import asyncio
from typing import TYPE_CHECKING
from unittest import mock
import pytest
@ -19,9 +18,6 @@ from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from tests.utils.decorators import coroutine_test
from tests.utils.robotstxt import rerp_available
if TYPE_CHECKING:
from scrapy.crawler import Crawler
class TestRobotsTxtMiddleware:
def setup_method(self) -> None:
@ -39,7 +35,7 @@ class TestRobotsTxtMiddleware:
with pytest.raises(NotConfigured):
RobotsTxtMiddleware(self.crawler)
def _get_successful_crawler(self) -> Crawler:
def _get_successful_crawler(self) -> mock.MagicMock:
crawler = self.crawler
crawler.settings.set("ROBOTSTXT_OBEY", True)
ROBOTS = """
@ -54,8 +50,8 @@ Disallow: /some/randome/page.html
""".encode()
response = TextResponse("http://site.local/robots.txt", body=ROBOTS)
async def return_response(request):
deferred = Deferred()
async def return_response(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.callback, response)
return await maybe_deferred_to_future(deferred)
@ -130,15 +126,15 @@ Disallow: /some/randome/page.html
Request("http://site.local/static/", meta=meta), middleware
)
def _get_garbage_crawler(self) -> Crawler:
def _get_garbage_crawler(self) -> mock.MagicMock:
crawler = self.crawler
crawler.settings.set("ROBOTSTXT_OBEY", True)
response = Response(
"http://site.local/robots.txt", body=b"GIF89a\xd3\x00\xfe\x00\xa2"
)
async def return_response(request):
deferred = Deferred()
async def return_response(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.callback, response)
return await maybe_deferred_to_future(deferred)
@ -154,13 +150,13 @@ Disallow: /some/randome/page.html
await self.assertNotIgnored(Request("http://site.local/admin/main"), middleware)
await self.assertNotIgnored(Request("http://site.local/static/"), middleware)
def _get_emptybody_crawler(self) -> Crawler:
def _get_emptybody_crawler(self) -> mock.MagicMock:
crawler = self.crawler
crawler.settings.set("ROBOTSTXT_OBEY", True)
response = Response("http://site.local/robots.txt")
async def return_response(request):
deferred = Deferred()
async def return_response(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.callback, response)
return await maybe_deferred_to_future(deferred)
@ -180,8 +176,8 @@ Disallow: /some/randome/page.html
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
err = CannotResolveHostError("Robotstxt address not found")
async def return_failure(request):
deferred = Deferred()
async def return_failure(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.errback, failure.Failure(err))
return await maybe_deferred_to_future(deferred)
@ -208,8 +204,8 @@ Disallow: /some/randome/page.html
async def test_ignore_robotstxt_request(self):
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
async def ignore_request(request):
deferred = Deferred()
async def ignore_request(request: Request) -> Response:
deferred: Deferred[Response] = Deferred()
call_later(0, deferred.errback, failure.Failure(IgnoreRequest()))
return await maybe_deferred_to_future(deferred)
@ -236,7 +232,7 @@ Disallow: /some/randome/page.html
@coroutine_test
async def test_robotstxt_local_file(self):
middleware = RobotsTxtMiddleware(self._get_emptybody_crawler())
middleware.process_request_2 = mock.MagicMock()
middleware.process_request_2 = mock.MagicMock() # type: ignore[method-assign]
await middleware.process_request(Request("data:text/plain,Hello World data"))
assert not middleware.process_request_2.called

72
tests/test_robotstxt_interface.py
View File

@ -1,3 +1,7 @@
from __future__ import annotations
from typing import TYPE_CHECKING
import pytest
from scrapy.robotstxt import (
@ -10,22 +14,32 @@ from scrapy.robotstxt import (
from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
from tests.utils.robotstxt import rerp_available
if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.crawler import Crawler
class BaseRobotParserTest:
def _setUp(self, parser_cls):
parser_cls: type[RobotParser]
def _setUp(self, parser_cls: type[RobotParser]) -> None:
self.parser_cls = parser_cls
def _parse(self, robotstxt_body: bytes) -> RobotParser:
# The parser backends only use the crawler to get the spider to log with.
return self.parser_cls.from_crawler(None, robotstxt_body) # type: ignore[arg-type]
def test_allowed(self):
robotstxt_robotstxt_body = (
b"User-agent: * \nDisallow: /disallowed \nAllow: /allowed \nCrawl-delay: 10"
)
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://www.site.local/allowed", "*")
assert not rp.allowed("https://www.site.local/disallowed", "*")
def test_allowed_wildcards(self):
def test_allowed_wildcards(self) -> None:
robotstxt_robotstxt_body = b"""User-agent: first
Disallow: /disallowed/*/end$
@ -33,9 +47,7 @@ class BaseRobotParserTest:
Allow: /*allowed
Disallow: /
"""
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://www.site.local/disallowed", "first")
assert not rp.allowed("https://www.site.local/disallowed/xyz/end", "first")
@ -46,23 +58,19 @@ class BaseRobotParserTest:
assert rp.allowed("https://www.site.local/is_still_allowed", "second")
assert rp.allowed("https://www.site.local/is_allowed_too", "second")
def test_length_based_precedence(self):
def test_length_based_precedence(self) -> None:
robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://www.site.local/page", "*")
def test_order_based_precedence(self):
def test_order_based_precedence(self) -> None:
robotstxt_robotstxt_body = b"User-agent: * \nDisallow: / \nAllow: /page"
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert not rp.allowed("https://www.site.local/page", "*")
def test_empty_response(self):
"""empty response should equal 'allow all'"""
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=b"")
rp = self._parse(b"")
assert rp.allowed("https://site.local/", "*")
assert rp.allowed("https://site.local/", "chrome")
assert rp.allowed("https://site.local/index.html", "*")
@ -71,9 +79,7 @@ class BaseRobotParserTest:
def test_garbage_response(self):
"""garbage response should be discarded, equal 'allow all'"""
robotstxt_robotstxt_body = b"GIF89a\xd3\x00\xfe\x00\xa2"
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://site.local/", "*")
assert rp.allowed("https://site.local/", "chrome")
assert rp.allowed("https://site.local/index.html", "*")
@ -81,12 +87,12 @@ class BaseRobotParserTest:
def test_crawl_delay(self):
robotstxt_body = b"User-agent: *\nDisallow: /private\nCrawl-delay: 10\n"
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
rp = self._parse(robotstxt_body)
assert rp.crawl_delay("*") == 10.0
def test_crawl_delay_unset(self):
robotstxt_body = b"User-agent: *\nDisallow: /private\n"
rp = self.parser_cls.from_crawler(crawler=None, robotstxt_body=robotstxt_body)
rp = self._parse(robotstxt_body)
assert rp.crawl_delay("*") is None
def test_unicode_url_and_useragent(self):
@ -100,9 +106,7 @@ class BaseRobotParserTest:
User-Agent: UnicödeBöt
Disallow: /some/randome/page.html""".encode()
rp = self.parser_cls.from_crawler(
crawler=None, robotstxt_body=robotstxt_robotstxt_body
)
rp = self._parse(robotstxt_robotstxt_body)
assert rp.allowed("https://site.local/", "*")
assert not rp.allowed("https://site.local/admin/", "*")
assert not rp.allowed("https://site.local/static/", "*")
@ -117,15 +121,13 @@ class TestRobotParser:
def test_crawl_delay_unsupported(self):
class AllowAllRobotParser(RobotParser):
@classmethod
def from_crawler(cls, crawler, robotstxt_body):
def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
return cls()
def allowed(self, url, user_agent):
def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
return True
rp = AllowAllRobotParser.from_crawler(
crawler=None, robotstxt_body=b"User-agent: *\nCrawl-delay: 10\n"
)
rp = AllowAllRobotParser()
assert rp.crawl_delay("*") is None
@ -162,21 +164,21 @@ class TestPythonRobotParser(BaseRobotParserTest):
not STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support length based directives precedence.",
)
def test_length_based_precedence(self):
def test_length_based_precedence(self) -> None:
super().test_length_based_precedence()
@pytest.mark.skipif(
STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support order based directives precedence.",
)
def test_order_based_precedence(self):
def test_order_based_precedence(self) -> None:
super().test_order_based_precedence()
@pytest.mark.skipif(
not STDLIB_IMPROVED_ROBOTFILEPARSER,
reason="RobotFileParser from this Python version does not support wildcards.",
)
def test_allowed_wildcards(self):
def test_allowed_wildcards(self) -> None:
super().test_allowed_wildcards()
@ -185,7 +187,7 @@ class TestRerpRobotParser(BaseRobotParserTest):
def setup_method(self):
super()._setUp(RerpRobotParser)
def test_length_based_precedence(self):
def test_length_based_precedence(self) -> None:
pytest.skip("Rerp does not support length based directives precedence.")
@ -193,5 +195,5 @@ class TestProtegoRobotParser(BaseRobotParserTest):
def setup_method(self):
super()._setUp(ProtegoRobotParser)
def test_order_based_precedence(self):
def test_order_based_precedence(self) -> None:
pytest.skip("Protego does not support order based directives precedence.")

98
tests/test_utils_console.py
View File

@ -1,10 +1,38 @@
from __future__ import annotations
import subprocess
import sys
from importlib.util import find_spec
from io import BytesIO
from typing import TYPE_CHECKING
import pytest
from pexpect import EOF
from scrapy.utils.console import get_shell_embed_func, start_python_console
from scrapy.utils.test import get_testenv
if TYPE_CHECKING:
from pathlib import Path
CONSOLE = """
from scrapy.utils.console import start_python_console
start_python_console(banner="SHELL-READY", shells=["ipython"])
"""
CONSOLE_IN_RUNNING_LOOP = """
import asyncio
from scrapy.utils.console import start_python_console
async def main():
start_python_console(banner="SHELL-READY", shells=["ipython"])
asyncio.run(main())
"""
def test_get_shell_embed_func():
@ -61,6 +89,76 @@ def test_get_shell_embed_func_default():
assert shell.__name__ == expected
@pytest.mark.skipif(find_spec("IPython") is None, reason="IPython is not installed")
class TestIPythonShell:
"""Starting an IPython shell, with and without an asyncio event loop already
running in the calling thread. The latter happens when inspect_response() is
called from a spider callback while using the asyncio reactor."""
@staticmethod
def _env(tmp_path: Path) -> dict[str, str]:
env = get_testenv()
# Keep IPython away from the profile and history of the user running the tests.
env["IPYTHONDIR"] = str(tmp_path)
return env
def test_simple_prompt(self, tmp_path: Path) -> None:
"""IPython falls back to its simple prompt, which needs no event loop,
when stdin is not a TTY."""
env = self._env(tmp_path)
p = subprocess.run(
[sys.executable, "-c", CONSOLE_IN_RUNNING_LOOP],
check=False,
capture_output=True,
encoding="utf-8",
timeout=60,
env=env,
stdin=subprocess.DEVNULL,
)
output = p.stdout + p.stderr
assert "SHELL-READY" in output
assert p.returncode == 0, output
@pytest.mark.skipif(
sys.platform == "win32", reason="requires a POSIX pseudo-terminal"
)
@pytest.mark.parametrize(
"script",
[CONSOLE, CONSOLE_IN_RUNNING_LOOP],
ids=["no_running_loop", "running_loop"],
)
def test_tty(self, tmp_path: Path, script: str) -> None:
"""IPython uses prompt_toolkit, which needs an event loop of its own,
when stdin is a TTY."""
# pexpect only defines spawn, which needs a pseudo-terminal, on POSIX.
from pexpect import spawn # noqa: PLC0415
env = self._env(tmp_path)
env.pop("IPY_TEST_SIMPLE_PROMPT", None)
env["TERM"] = "xterm"
logfile = BytesIO()
p = spawn(
sys.executable,
["-c", script],
env=env,
timeout=60,
)
p.logfile_read = logfile
try:
# Wait for the prompt, which prompt_toolkit draws once it is done
# querying the terminal, before typing into it.
p.expect(r"In \[")
p.sendline("21*2")
p.expect_exact("42")
p.sendline("exit()")
p.expect(EOF)
finally:
p.close()
output = logfile.getvalue().decode()
assert "Traceback" not in output
assert p.exitstatus == 0, output
def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None:
def embed(namespace: dict[str, object], banner: str) -> None:
raise SystemExit

7
tox.ini
View File

@ -5,7 +5,8 @@
[tox]
requires =
sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.9
sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.10
tox-uv
envlist =
pre-commit
pylint
@ -190,8 +191,8 @@ deps =
brotlicffi==1.2.0.0; implementation_name == "pypy"
google-cloud-storage==1.29.0
httpx2[http2,socks]==2.0.0
ipython==7.1.0
ptpython==2.0.1
ipython==8.15.0
ptpython==3.0.23
robotexclusionrulesparser==1.6.2
uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"
zstandard==0.16.0; implementation_name != "pypy"