From 590955fac8de5d1f951b4ecb724eb2ea4f212653 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 15 Mar 2023 16:03:44 +0100
Subject: [PATCH] Provide separate test environments for botocore and boto3
 extras; add extra-deps-pinned and remote extras from pinned

---
 .github/workflows/tests-ubuntu.yml |  9 ++++++
 tox.ini                            | 44 ++++++++++++++++++++++++------
 2 files changed, 45 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 8fcf90a18..96b26a1f8 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -37,10 +37,19 @@ jobs:
         - python-version: pypy3.7
           env:
             TOXENV: pypy3-pinned
+        - python-version: 3.7.13
+          env:
+            TOXENV: extra-deps-pinned
+        - python-version: 3.7.13
+          env:
+            TOXENV: botocore-pinned
 
         - python-version: "3.11"
           env:
             TOXENV: extra-deps
+        - python-version: "3.11"
+          env:
+            TOXENV: botocore
 
     steps:
     - uses: actions/checkout@v3
diff --git a/tox.ini b/tox.ini
index 5c2f583d9..f94d7f751 100644
--- a/tox.ini
+++ b/tox.ini
@@ -88,11 +88,6 @@ deps =
 
     # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
     # above, hence we do not install it in pinned environments at the moment
-
-    # Extras
-    botocore==1.4.87
-    google-cloud-storage==1.29.0
-    Pillow==7.1.0
 setenv =
     _SCRAPY_PINNED=true
 install_command =
@@ -119,14 +114,26 @@ setenv =
 basepython = python3
 deps =
     {[testenv]deps}
-    botocore>=1.4.87
+    boto3
     google-cloud-storage
     # Twisted[http2] currently forces old mitmproxy because of h2 version
     # restrictions in their deps, so we need to pin old markupsafe here too.
     markupsafe < 2.1.0
     robotexclusionrulesparser
-    Pillow>=4.0.0
-    Twisted[http2]>=17.9.0
+    Pillow
+    Twisted[http2]
+
+[testenv:extra-deps-pinned]
+basepython = python3.7
+deps =
+    {[pinned]deps}
+    boto3==1.0.0
+    google-cloud-storage==1.29.0
+    Pillow==7.1.0
+    robotexclusionrulesparser==1.6.2
+install_command = {[pinned]install_command}
+setenv =
+    {[pinned]setenv}
 
 [testenv:asyncio]
 commands =
@@ -185,3 +192,24 @@ deps = {[docs]deps}
 setenv = {[docs]setenv}
 commands =
     sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
+
+
+# Run S3 tests with botocore installed but without boto3.
+
+[testenv:botocore]
+deps =
+    {[testenv]deps}
+    botocore>=1.4.87
+commands =
+    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
+
+[testenv:botocore-pinned]
+basepython = python3.7
+deps =
+    {[pinned]deps}
+    botocore==1.4.87
+install_command = {[pinned]install_command}
+setenv =
+    {[pinned]setenv}
+commands =
+    pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}