mirror of https://github.com/scrapy/scrapy.git
322 lines
7.9 KiB
INI
322 lines
7.9 KiB
INI
# Tox (https://tox.readthedocs.io/) is a tool for running tests
|
|
# in multiple virtualenvs. This configuration file will run the
|
|
# test suite on all supported python versions. To use it, "pip install tox"
|
|
# and then run "tox" from this directory.
|
|
|
|
[tox]
|
|
requires =
|
|
sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.8
|
|
envlist =
|
|
pre-commit
|
|
pylint
|
|
mypy
|
|
mypy-tests
|
|
twinecheck
|
|
docs
|
|
docs-tests
|
|
docs-links
|
|
docs-coverage
|
|
min
|
|
min-extra-deps
|
|
min-default-reactor
|
|
min-no-reactor
|
|
min-botocore
|
|
min-pypy3
|
|
py{310,311,312,313,314}
|
|
extra-deps
|
|
default-reactor
|
|
no-reactor
|
|
no-reactor-extra-deps
|
|
botocore
|
|
pypy3
|
|
pypy3-extra-deps
|
|
minversion = 1.7.0
|
|
|
|
[test-requirements]
|
|
deps =
|
|
attrs
|
|
coverage >= 7.10.6
|
|
httpx
|
|
pexpect >= 4.8.0
|
|
pyftpdlib >= 2.0.1
|
|
pygments
|
|
pytest
|
|
pytest-cov >= 7.0.0
|
|
pytest-xdist
|
|
sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
|
|
testfixtures
|
|
pytest-twisted >= 1.14.3
|
|
|
|
[testenv]
|
|
deps =
|
|
{[test-requirements]deps}
|
|
pytest >= 8.4.1 # https://github.com/pytest-dev/pytest/pull/13502
|
|
passenv =
|
|
PYTHONTRACEMALLOC
|
|
PYTEST_ADDOPTS
|
|
MITMDUMP
|
|
S3_TEST_FILE_URI
|
|
AWS_ACCESS_KEY_ID
|
|
AWS_SECRET_ACCESS_KEY
|
|
AWS_SESSION_TOKEN
|
|
GCS_TEST_FILE_URI
|
|
GCS_PROJECT_ID
|
|
#allow tox virtualenv to upgrade pip/wheel/setuptools
|
|
download = true
|
|
commands =
|
|
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 scrapy tests --doctest-modules}
|
|
|
|
[testenv:mypy]
|
|
basepython = python3.10
|
|
deps =
|
|
mypy==2.1.0
|
|
typing-extensions==4.16.0
|
|
Pillow==12.3.0
|
|
Protego==0.6.2
|
|
Twisted==26.4.0
|
|
attrs==26.1.0
|
|
boto3-stubs[s3]==1.43.41
|
|
botocore-stubs==1.43.14
|
|
h2==4.3.0
|
|
httpx==0.28.1
|
|
itemadapter==0.13.1
|
|
ptpython==3.0.32
|
|
# newer ones require newer Python
|
|
ipython==8.39.0
|
|
pyOpenSSL==26.3.0
|
|
pytest==9.1.1
|
|
socksio==1.0.0
|
|
types-Pygments==2.20.0.20260518
|
|
types-defusedxml==0.7.0.20260504
|
|
types-lxml==2026.2.16
|
|
types-pexpect==4.9.0.20260518
|
|
uvloop==0.22.1
|
|
w3lib==2.4.1
|
|
zstandard==0.25.0
|
|
commands =
|
|
mypy {posargs:scrapy tests}
|
|
|
|
[testenv:mypy-tests]
|
|
basepython = python3.10
|
|
deps =
|
|
{[test-requirements]deps}
|
|
{[testenv:mypy]deps}
|
|
pytest-mypy-testing==0.2.0
|
|
commands =
|
|
pytest {posargs:tests_typing}
|
|
|
|
[testenv:pre-commit]
|
|
basepython = python3
|
|
deps =
|
|
pre-commit
|
|
commands =
|
|
pre-commit run {posargs:--all-files}
|
|
|
|
[testenv:pylint]
|
|
basepython = python3
|
|
deps =
|
|
{[testenv:extra-deps]deps}
|
|
pylint==4.0.6
|
|
pylint-per-file-ignores # https://github.com/pylint-dev/pylint/issues/3767#issuecomment-1319916278
|
|
commands =
|
|
pylint {posargs:conftest.py docs extras scrapy tests}
|
|
|
|
[testenv:twinecheck]
|
|
basepython = python3
|
|
deps =
|
|
twine==6.2.0
|
|
build==1.5.0
|
|
commands =
|
|
python -m build --sdist
|
|
twine check dist/*
|
|
|
|
[min]
|
|
basepython = python3.10
|
|
deps =
|
|
# pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0
|
|
pytest==8.4.0
|
|
Protego==0.1.15
|
|
Twisted==21.7.0
|
|
cryptography==37.0.0
|
|
cssselect==0.9.1
|
|
httpx==0.27.1
|
|
itemadapter==0.1.0
|
|
lxml==4.6.4
|
|
parsel==1.5.0
|
|
pyOpenSSL==22.0.0
|
|
queuelib==1.4.2
|
|
service_identity==23.1.0
|
|
w3lib==1.17.0
|
|
zope.interface==5.1.0
|
|
{[test-requirements]deps}
|
|
setenv =
|
|
_SCRAPY_MIN=true
|
|
commands =
|
|
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=min.junit.xml -o junit_family=legacy --durations=10 scrapy tests}
|
|
|
|
[testenv:min]
|
|
basepython = {[min]basepython}
|
|
deps =
|
|
{[min]deps}
|
|
PyDispatcher==2.0.5
|
|
setenv =
|
|
{[min]setenv}
|
|
commands = {[min]commands}
|
|
|
|
[testenv:extra-deps]
|
|
basepython = python3
|
|
deps =
|
|
{[testenv]deps}
|
|
Pillow
|
|
Twisted[http2]
|
|
boto3
|
|
bpython # optional for shell wrapper tests
|
|
brotli >= 1.2.0; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests
|
|
brotlicffi >= 1.2.0.0; implementation_name == "pypy" # optional for HTTP compress downloader middleware tests
|
|
google-cloud-storage
|
|
httpx[http2,socks]
|
|
ipython
|
|
robotexclusionrulesparser
|
|
uvloop; platform_system != "Windows" and implementation_name != "pypy"
|
|
zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests
|
|
|
|
[testenv:min-extra-deps]
|
|
basepython = {[min]basepython}
|
|
deps =
|
|
{[min]deps}
|
|
Pillow==8.3.2
|
|
Twisted[http2]==21.7.0
|
|
boto3==1.20.0
|
|
bpython==0.7.1
|
|
brotli==1.2.0; implementation_name != "pypy"
|
|
brotlicffi==1.2.0.0; implementation_name == "pypy"
|
|
google-cloud-storage==1.29.0
|
|
httpx==0.27.1
|
|
ipython==7.1.0
|
|
robotexclusionrulesparser==1.6.2
|
|
uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"
|
|
zstandard==0.16.0; implementation_name != "pypy"
|
|
setenv =
|
|
{[min]setenv}
|
|
commands = {[min]commands}
|
|
|
|
[testenv:default-reactor]
|
|
commands =
|
|
{[testenv]commands} --reactor=default
|
|
|
|
[testenv:min-default-reactor]
|
|
basepython = {[min]basepython}
|
|
deps = {[testenv:min]deps}
|
|
commands = {[min]commands} --reactor=default
|
|
setenv =
|
|
{[min]setenv}
|
|
|
|
[testenv:no-reactor]
|
|
deps =
|
|
{[testenv]deps}
|
|
pytest-asyncio
|
|
commands =
|
|
{[testenv]commands} -p no:twisted --reactor=none
|
|
|
|
[testenv:no-reactor-extra-deps]
|
|
deps =
|
|
{[testenv:extra-deps]deps}
|
|
pytest-asyncio
|
|
commands =
|
|
{[testenv]commands} -p no:twisted --reactor=none
|
|
|
|
[testenv:min-no-reactor]
|
|
basepython = {[min]basepython}
|
|
deps =
|
|
{[testenv:min]deps}
|
|
pytest-asyncio
|
|
commands = {[min]commands} -p no:twisted --reactor=none
|
|
setenv =
|
|
{[min]setenv}
|
|
|
|
[testenv:pypy3]
|
|
basepython = pypy3
|
|
commands =
|
|
; not enabling coverage as it significantly increases the run time
|
|
pytest {posargs:--durations=10 scrapy tests}
|
|
|
|
[testenv:pypy3-extra-deps]
|
|
basepython = pypy3
|
|
deps =
|
|
{[testenv:extra-deps]deps}
|
|
commands = {[testenv:pypy3]commands}
|
|
|
|
[testenv:min-pypy3]
|
|
basepython = pypy3.11
|
|
deps =
|
|
PyPyDispatcher==2.1.0
|
|
{[test-requirements]deps}
|
|
pytest==8.4.0
|
|
Protego==0.1.15
|
|
Twisted==21.7.0
|
|
cryptography==44.0.2
|
|
cssselect==0.9.1
|
|
itemadapter==0.1.0
|
|
lxml==5.3.2
|
|
parsel==1.5.0
|
|
pyOpenSSL==24.3.0
|
|
queuelib==1.4.2
|
|
service_identity==23.1.0
|
|
# w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses
|
|
# an inline flag placement that Python 3.11 treats as an error: global
|
|
# flags not at the start of the expression. w3lib 1.18 stopped encoding []
|
|
# in URLs until 2.1.0 brought that behavior back. Tests for verbatim_url
|
|
# rely on that encoding.
|
|
w3lib==2.1.0
|
|
zope.interface==5.1.0
|
|
commands =
|
|
; disabling coverage
|
|
pytest {posargs:--durations=10 scrapy tests}
|
|
setenv =
|
|
{[min]setenv}
|
|
|
|
[testenv:docs-tests]
|
|
changedir = docs
|
|
deps =
|
|
{[test-requirements]deps}
|
|
-rdocs/requirements.txt
|
|
form2request
|
|
commands =
|
|
pytest
|
|
|
|
[testenv:docs-coverage]
|
|
changedir = docs
|
|
deps =
|
|
-rdocs/requirements.txt
|
|
commands =
|
|
sphinx-build -b coverage . {envtmpdir}/coverage
|
|
|
|
[testenv:docs-links]
|
|
changedir = docs
|
|
deps =
|
|
-rdocs/requirements.txt
|
|
commands =
|
|
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
|
|
|
|
|
|
# Run S3 tests with botocore installed but without boto3.
|
|
|
|
[testenv:botocore]
|
|
deps =
|
|
{[testenv]deps}
|
|
botocore>=1.13.45
|
|
commands =
|
|
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_botocore
|
|
|
|
[testenv:min-botocore]
|
|
basepython = {[min]basepython}
|
|
deps =
|
|
{[min]deps}
|
|
botocore==1.13.45
|
|
# botocore 1.13.45 requires urllib3>=1.20,<1.26; requests 2.33.0 requires urllib3>=1.26,<3
|
|
requests<2.33.0
|
|
setenv =
|
|
{[min]setenv}
|
|
commands =
|
|
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore
|