diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 8fcf90a18..96b26a1f8 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -37,10 +37,19 @@ jobs: - python-version: pypy3.7 env: TOXENV: pypy3-pinned + - python-version: 3.7.13 + env: + TOXENV: extra-deps-pinned + - python-version: 3.7.13 + env: + TOXENV: botocore-pinned - python-version: "3.11" env: TOXENV: extra-deps + - python-version: "3.11" + env: + TOXENV: botocore steps: - uses: actions/checkout@v3 diff --git a/tox.ini b/tox.ini index 5c2f583d9..f94d7f751 100644 --- a/tox.ini +++ b/tox.ini @@ -88,11 +88,6 @@ deps = # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment - - # Extras - botocore==1.4.87 - google-cloud-storage==1.29.0 - Pillow==7.1.0 setenv = _SCRAPY_PINNED=true install_command = @@ -119,14 +114,26 @@ setenv = basepython = python3 deps = {[testenv]deps} - botocore>=1.4.87 + boto3 google-cloud-storage # Twisted[http2] currently forces old mitmproxy because of h2 version # restrictions in their deps, so we need to pin old markupsafe here too. markupsafe < 2.1.0 robotexclusionrulesparser - Pillow>=4.0.0 - Twisted[http2]>=17.9.0 + Pillow + Twisted[http2] + +[testenv:extra-deps-pinned] +basepython = python3.7 +deps = + {[pinned]deps} + boto3==1.0.0 + google-cloud-storage==1.29.0 + Pillow==7.1.0 + robotexclusionrulesparser==1.6.2 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:asyncio] commands = @@ -185,3 +192,24 @@ deps = {[docs]deps} setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck + + +# Run S3 tests with botocore installed but without boto3. + +[testenv:botocore] +deps = + {[testenv]deps} + botocore>=1.4.87 +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} + +[testenv:botocore-pinned] +basepython = python3.7 +deps = + {[pinned]deps} + botocore==1.4.87 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}