Provide separate test environments for botocore and boto3 extras; add extra-deps-pinned and remote extras from pinned

.github/workflows/tests-ubuntu.yml

@@ -37,10 +37,19 @@ jobs:
         - python-version: pypy3.7
           env:
             TOXENV: pypy3-pinned
+        - python-version: 3.7.13
+          env:
+            TOXENV: extra-deps-pinned
+        - python-version: 3.7.13
+          env:
+            TOXENV: botocore-pinned
 
         - python-version: "3.11"
           env:
             TOXENV: extra-deps
+        - python-version: "3.11"
+          env:
+            TOXENV: botocore
 
     steps:
     - uses: actions/checkout@v3

tox.ini

@@ -88,11 +88,6 @@ deps =
 
     # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
     # above, hence we do not install it in pinned environments at the moment
-
-    # Extras
-    botocore==1.4.87
-    google-cloud-storage==1.29.0
-    Pillow==7.1.0
 setenv =
     _SCRAPY_PINNED=true
 install_command =
@@ -119,14 +114,26 @@ setenv =
 basepython = python3
 deps =
     {[testenv]deps}
-    botocore>=1.4.87
+    boto3
     google-cloud-storage
     # Twisted[http2] currently forces old mitmproxy because of h2 version
     # restrictions in their deps, so we need to pin old markupsafe here too.
     markupsafe < 2.1.0
     robotexclusionrulesparser
-    Pillow>=4.0.0
+    Pillow
-    Twisted[http2]>=17.9.0
+    Twisted[http2]
+
+[testenv:extra-deps-pinned]
+basepython = python3.7
+deps =
+    {[pinned]deps}
+    boto3==1.0.0
+    google-cloud-storage==1.29.0
+    Pillow==7.1.0
+    robotexclusionrulesparser==1.6.2
+install_command = {[pinned]install_command}
+setenv =
+    {[pinned]setenv}
 
 [testenv:asyncio]
 commands =
@@ -185,3 +192,24 @@ deps = {[docs]deps}
 setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
+
+
+# Run S3 tests with botocore installed but without boto3.
+
+[testenv:botocore]
+deps =
+    {[testenv]deps}
+    botocore>=1.4.87
+commands =
+    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
+
+[testenv:botocore-pinned]
+basepython = python3.7
+deps =
+    {[pinned]deps}
+    botocore==1.4.87
+install_command = {[pinned]install_command}
+setenv =
+    {[pinned]setenv}
+commands =
+    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}