From 60bf56b715e443951125a10ff91ad1699270a82c Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 15 Feb 2023 12:15:24 +0100 Subject: [PATCH 01/14] Add boto3 availability util method --- scrapy/utils/boto.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 085ee7d25..7b18b6bcf 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -8,3 +8,12 @@ def is_botocore_available(): return True except ImportError: return False + + +def is_boto3_available(): + try: + import boto3 # noqa: F401 + + return True + except ImportError: + return False From 01ad49515d31e3053949d20e923dc61a87147eab Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 15 Feb 2023 12:18:28 +0100 Subject: [PATCH 02/14] Use boto3 session and client --- scrapy/extensions/feedexport.py | 37 ++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index cd26b5778..34eb8e4b4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -173,16 +173,37 @@ class S3FeedStorage(BlockingFeedStorage): self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url - import botocore.session + if is_boto3_available(): + import boto3.session + session = boto3.session.Session() + + self.s3_client = session.client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url, + ) + else: + warnings.warn( + "Botocore usage is deprecated for S3FeedStorage, " + "please use boto3 to avoid problems", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + import botocore.session + session = botocore.get_session() session = botocore.session.get_session() - self.s3_client = session.create_client( - "s3", - aws_access_key_id=self.access_key, - aws_secret_access_key=self.secret_key, - aws_session_token=self.session_token, - endpoint_url=self.endpoint_url, - ) + self.s3_client = session.create_client( + "s3", + aws_access_key_id=self.access_key, + aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, + endpoint_url=self.endpoint_url, + ) + if feed_options and feed_options.get("overwrite", True) is False: logger.warning( "S3 does not support appending to files. To " From c1a8baa1fa0c6210e5c1351ab5a4063b8dacaa7b Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 15 Feb 2023 12:20:01 +0100 Subject: [PATCH 03/14] Add forgotten import --- scrapy/extensions/feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 34eb8e4b4..0e05337bc 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -22,6 +22,7 @@ from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager from scrapy.utils.boto import is_botocore_available +from scrapy.utils.boto import is_boto3_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info @@ -195,7 +196,6 @@ class S3FeedStorage(BlockingFeedStorage): import botocore.session session = botocore.get_session() - session = botocore.session.get_session() self.s3_client = session.create_client( "s3", aws_access_key_id=self.access_key, From 59ba3c4e4cabd661810e0ac1b9963040d2e42f31 Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 15 Feb 2023 16:29:06 +0100 Subject: [PATCH 04/14] Use boto3's `upload_fileobj` --- scrapy/extensions/feedexport.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 0e05337bc..601fde7eb 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -174,7 +174,9 @@ class S3FeedStorage(BlockingFeedStorage): self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url - if is_boto3_available(): + self._using_boto3 = is_boto3_available() + + if self._using_boto3: import boto3.session session = boto3.session.Session() @@ -187,8 +189,8 @@ class S3FeedStorage(BlockingFeedStorage): ) else: warnings.warn( - "Botocore usage is deprecated for S3FeedStorage, " - "please use boto3 to avoid problems", + "`botocore` usage has been deprecated for S3 feed " + "export, please use `boto3` to avoid problems", category=ScrapyDeprecationWarning, stacklevel=2, ) @@ -227,9 +229,14 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file): file.seek(0) kwargs = {"ACL": self.acl} if self.acl else {} - self.s3_client.put_object( - Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs - ) + if self._using_boto3: + self.s3_client.upload_fileobj( + Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs + ) + else: + self.s3_client.put_object( + Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs + ) file.close() From 29c2477f0a8365d0476fb8c07f391fa109f4615a Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 15 Feb 2023 16:40:05 +0100 Subject: [PATCH 05/14] Document the need to install boto3 --- docs/topics/feed-exports.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index eef0bb5ca..8aa3e3be4 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -101,12 +101,12 @@ The storages backends supported out of the box are: - :ref:`topics-feed-storage-fs` - :ref:`topics-feed-storage-ftp` -- :ref:`topics-feed-storage-s3` (requires botocore_) +- :ref:`topics-feed-storage-s3` (requires boto3_) - :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_) - :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are -not available. For example, the S3 backend is only available if the botocore_ +not available. For example, the S3 backend is only available if at least the botocore_ library is installed. @@ -193,7 +193,7 @@ The feeds are stored on `Amazon S3`_. - ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` -- Required external libraries: `botocore`_ >= 1.4.87 +- Required external libraries: `boto3`_ >= 1.26.70, will fall back to botocore_ if unavailable The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: @@ -779,6 +779,7 @@ source spider in the feed URI: .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ +.. _boto3: https://github.com/boto/boto3 .. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _Google Cloud Storage: https://cloud.google.com/storage/ From f45a7d3f3c90ad27b2b13bd1ee038ce2fe9c02f7 Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 15 Feb 2023 17:07:55 +0100 Subject: [PATCH 06/14] Remove `stacklevel` from warning --- scrapy/extensions/feedexport.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 601fde7eb..e27e6f915 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -192,7 +192,6 @@ class S3FeedStorage(BlockingFeedStorage): "`botocore` usage has been deprecated for S3 feed " "export, please use `boto3` to avoid problems", category=ScrapyDeprecationWarning, - stacklevel=2, ) import botocore.session From eb0cca471d20f00a1cc0980b84fe6266772faaa3 Mon Sep 17 00:00:00 2001 From: jazzthief Date: Thu, 16 Feb 2023 16:53:49 +0100 Subject: [PATCH 07/14] Apply pre-commit changes --- scrapy/extensions/feedexport.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index e27e6f915..36107a62b 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -21,8 +21,7 @@ from zope.interface import Interface, implementer from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager -from scrapy.utils.boto import is_botocore_available -from scrapy.utils.boto import is_boto3_available +from scrapy.utils.boto import is_boto3_available, is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info @@ -178,6 +177,7 @@ class S3FeedStorage(BlockingFeedStorage): if self._using_boto3: import boto3.session + session = boto3.session.Session() self.s3_client = session.client( @@ -195,6 +195,7 @@ class S3FeedStorage(BlockingFeedStorage): ) import botocore.session + session = botocore.get_session() self.s3_client = session.create_client( From 45b9dbae40de00fc7d4498e871538c074c64be8f Mon Sep 17 00:00:00 2001 From: jazzthief Date: Wed, 22 Feb 2023 13:28:34 +0100 Subject: [PATCH 08/14] Fix a typo --- scrapy/extensions/feedexport.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 36107a62b..c4ec410e3 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -196,7 +196,7 @@ class S3FeedStorage(BlockingFeedStorage): import botocore.session - session = botocore.get_session() + session = botocore.session.get_session() self.s3_client = session.create_client( "s3", From 39dbfa1d8276e1c8abb6aedc51f644ee27264f90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Mar 2023 15:31:39 +0100 Subject: [PATCH 09/14] Minimize test reliance on S3; do not install botocore on the default test environments --- tests/test_feedexport.py | 23 ++++++----------------- tests/test_pipeline_files.py | 12 ++++++++---- tox.ini | 4 +--- 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index eafe1b334..19ca311c3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -235,8 +235,10 @@ class BlockingFeedStorageTest(unittest.TestCase): class S3FeedStorageTest(unittest.TestCase): - def test_parse_credentials(self): + def setUp(self): skip_if_no_boto() + + def test_parse_credentials(self): aws_credentials = { "AWS_ACCESS_KEY_ID": "settings_key", "AWS_SECRET_ACCESS_KEY": "settings_secret", @@ -272,8 +274,6 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store(self): - skip_if_no_boto() - settings = { "AWS_ACCESS_KEY_ID": "access_key", "AWS_SECRET_ACCESS_KEY": "secret_key", @@ -392,7 +392,6 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_without_acl(self): - skip_if_no_boto() storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", @@ -408,7 +407,6 @@ class S3FeedStorageTest(unittest.TestCase): @defer.inlineCallbacks def test_store_botocore_with_acl(self): - skip_if_no_boto() storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) @@ -888,15 +886,10 @@ class FeedExportTest(FeedExportTestBase): @defer.inlineCallbacks def test_stats_multiple_file(self): settings = { - "AWS_ACCESS_KEY_ID": "access_key", - "AWS_SECRET_ACCESS_KEY": "secret_key", "FEEDS": { printf_escape(path_to_url(str(self._random_temp_filename()))): { "format": "json", }, - "s3://bucket/key/foo.csv": { - "format": "csv", - }, "stdout:": { "format": "xml", }, @@ -908,18 +901,12 @@ class FeedExportTest(FeedExportTestBase): self.assertIn( "feedexport/success_count/FileFeedStorage", crawler.stats.get_stats() ) - self.assertIn( - "feedexport/success_count/S3FeedStorage", crawler.stats.get_stats() - ) self.assertIn( "feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats() ) self.assertEqual( crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1 ) - self.assertEqual( - crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1 - ) self.assertEqual( crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1 ) @@ -2535,7 +2522,6 @@ class BatchDeliveriesTest(FeedExportTestBase): @defer.inlineCallbacks def test_s3_export(self): skip_if_no_boto() - bucket = "mybucket" items = [ self.MyItem({"foo": "bar1", "egg": "spam1"}), @@ -2707,6 +2693,9 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase): maxDiff = None + def setUp(self): + skip_if_no_boto() + def test_init(self): settings_dict = { "FEED_URI": "file:///tmp/foobar", diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 9701e5d4e..e0bcfcfea 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -225,12 +225,16 @@ class FilesPipelineTestCase(unittest.TestCase): class FilesPipelineTestCaseFieldsMixin: + def setUp(self): + self.tempdir = mkdtemp() + + def tearDown(self): + rmtree(self.tempdir) + def test_item_fields_default(self): url = "http://www.example.com/files/1.txt" item = self.item_class(name="item1", file_urls=[url]) - pipeline = FilesPipeline.from_settings( - Settings({"FILES_STORE": "s3://example/files/"}) - ) + pipeline = FilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir})) requests = list(pipeline.get_media_requests(item, None)) self.assertEqual(requests[0].url, url) results = [(True, {"url": url})] @@ -245,7 +249,7 @@ class FilesPipelineTestCaseFieldsMixin: pipeline = FilesPipeline.from_settings( Settings( { - "FILES_STORE": "s3://example/files/", + "FILES_STORE": self.tempdir, "FILES_URLS_FIELD": "custom_file_urls", "FILES_RESULT_FIELD": "custom_files", } diff --git a/tox.ini b/tox.ini index 5a9d9cf29..5c2f583d9 100644 --- a/tox.ini +++ b/tox.ini @@ -18,8 +18,6 @@ deps = mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy' # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower) markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy' - # Extras - botocore>=1.4.87 passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID @@ -121,7 +119,7 @@ setenv = basepython = python3 deps = {[testenv]deps} - boto + botocore>=1.4.87 google-cloud-storage # Twisted[http2] currently forces old mitmproxy because of h2 version # restrictions in their deps, so we need to pin old markupsafe here too. From 590955fac8de5d1f951b4ecb724eb2ea4f212653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Mar 2023 16:03:44 +0100 Subject: [PATCH 10/14] Provide separate test environments for botocore and boto3 extras; add extra-deps-pinned and remote extras from pinned --- .github/workflows/tests-ubuntu.yml | 9 ++++++ tox.ini | 44 ++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 8fcf90a18..96b26a1f8 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -37,10 +37,19 @@ jobs: - python-version: pypy3.7 env: TOXENV: pypy3-pinned + - python-version: 3.7.13 + env: + TOXENV: extra-deps-pinned + - python-version: 3.7.13 + env: + TOXENV: botocore-pinned - python-version: "3.11" env: TOXENV: extra-deps + - python-version: "3.11" + env: + TOXENV: botocore steps: - uses: actions/checkout@v3 diff --git a/tox.ini b/tox.ini index 5c2f583d9..f94d7f751 100644 --- a/tox.ini +++ b/tox.ini @@ -88,11 +88,6 @@ deps = # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies # above, hence we do not install it in pinned environments at the moment - - # Extras - botocore==1.4.87 - google-cloud-storage==1.29.0 - Pillow==7.1.0 setenv = _SCRAPY_PINNED=true install_command = @@ -119,14 +114,26 @@ setenv = basepython = python3 deps = {[testenv]deps} - botocore>=1.4.87 + boto3 google-cloud-storage # Twisted[http2] currently forces old mitmproxy because of h2 version # restrictions in their deps, so we need to pin old markupsafe here too. markupsafe < 2.1.0 robotexclusionrulesparser - Pillow>=4.0.0 - Twisted[http2]>=17.9.0 + Pillow + Twisted[http2] + +[testenv:extra-deps-pinned] +basepython = python3.7 +deps = + {[pinned]deps} + boto3==1.0.0 + google-cloud-storage==1.29.0 + Pillow==7.1.0 + robotexclusionrulesparser==1.6.2 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} [testenv:asyncio] commands = @@ -185,3 +192,24 @@ deps = {[docs]deps} setenv = {[docs]setenv} commands = sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck + + +# Run S3 tests with botocore installed but without boto3. + +[testenv:botocore] +deps = + {[testenv]deps} + botocore>=1.4.87 +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} + +[testenv:botocore-pinned] +basepython = python3.7 +deps = + {[pinned]deps} + botocore==1.4.87 +install_command = {[pinned]install_command} +setenv = + {[pinned]setenv} +commands = + pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3} From 7e7b41c6b32a639395cf3183e98fa8a8a78afd98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 15 Mar 2023 16:38:13 +0100 Subject: [PATCH 11/14] Fix test expectations for boto3 --- scrapy/extensions/feedexport.py | 3 +- tests/test_feedexport.py | 72 +++++++++++++++++++++------------ 2 files changed, 48 insertions(+), 27 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index c4ec410e3..4f0a946de 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -228,12 +228,13 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file): file.seek(0) - kwargs = {"ACL": self.acl} if self.acl else {} if self._using_boto3: + kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} self.s3_client.upload_fileobj( Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs ) else: + kwargs = {"ACL": self.acl} if self.acl else {} self.s3_client.put_object( Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs ) diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 19ca311c3..2e350df65 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -44,6 +44,7 @@ from scrapy.extensions.feedexport import ( S3FeedStorage, StdoutFeedStorage, _FeedSlot, + is_boto3_available, ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -285,30 +286,39 @@ class S3FeedStorageTest(unittest.TestCase): verifyObject(IFeedStorage, storage) file = mock.MagicMock() - from botocore.stub import Stubber - - with Stubber(storage.s3_client) as stub: - stub.add_response( - "put_object", - expected_params={ - "Body": file, - "Bucket": bucket, - "Key": key, - }, - service_response={}, - ) + if is_boto3_available(): + storage.s3_client = mock.MagicMock() yield storage.store(file) - - stub.assert_no_pending_responses() self.assertEqual( - file.method_calls, - [ - mock.call.seek(0), - # The call to read does not happen with Stubber - mock.call.close(), - ], + storage.s3_client.upload_fileobj.call_args, + mock.call(Bucket=bucket, Key=key, Fileobj=file), ) + else: + from botocore.stub import Stubber + + with Stubber(storage.s3_client) as stub: + stub.add_response( + "put_object", + expected_params={ + "Body": file, + "Bucket": bucket, + "Key": key, + }, + service_response={}, + ) + + yield storage.store(file) + + stub.assert_no_pending_responses() + self.assertEqual( + file.method_calls, + [ + mock.call.seek(0), + # The call to read does not happen with Stubber + mock.call.close(), + ], + ) def test_init_without_acl(self): storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key") @@ -391,7 +401,7 @@ class S3FeedStorageTest(unittest.TestCase): self.assertEqual(storage.endpoint_url, "https://example.com") @defer.inlineCallbacks - def test_store_botocore_without_acl(self): + def test_store_without_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", @@ -403,10 +413,18 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - self.assertNotIn("ACL", storage.s3_client.put_object.call_args[1]) + if is_boto3_available(): + acl = ( + storage.s3_client.upload_fileobj.call_args[1] + .get("ExtraArgs", {}) + .get("ACL") + ) + else: + acl = storage.s3_client.put_object.call_args[1].get("ACL") + self.assertIsNone(acl) @defer.inlineCallbacks - def test_store_botocore_with_acl(self): + def test_store_with_acl(self): storage = S3FeedStorage( "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl" ) @@ -416,9 +434,11 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - self.assertEqual( - storage.s3_client.put_object.call_args[1].get("ACL"), "custom-acl" - ) + if is_boto3_available(): + acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] + else: + acl = storage.s3_client.put_object.call_args[1]["ACL"] + self.assertEqual(acl, "custom-acl") def test_overwrite_default(self): with LogCapture() as log: From a17d996da2dad6d250dd93da34b5b63f3d63239d Mon Sep 17 00:00:00 2001 From: jazzthief Date: Thu, 16 Mar 2023 17:20:22 +0100 Subject: [PATCH 12/14] Change `boto3` version to 1.20.0 for `extra-deps-pinned` env --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index f94d7f751..80fc28735 100644 --- a/tox.ini +++ b/tox.ini @@ -127,7 +127,7 @@ deps = basepython = python3.7 deps = {[pinned]deps} - boto3==1.0.0 + boto3==1.20.0 google-cloud-storage==1.29.0 Pillow==7.1.0 robotexclusionrulesparser==1.6.2 From 4ebc08ef1042cafd16c6a7eb20e3a4dcf43e3c97 Mon Sep 17 00:00:00 2001 From: jazzthief Date: Thu, 16 Mar 2023 17:24:11 +0100 Subject: [PATCH 13/14] Switch from `is_boto3_available()` to `IS_BOTO3_AVAILABLE` var --- scrapy/extensions/feedexport.py | 14 ++++++++++---- scrapy/utils/boto.py | 9 --------- tests/test_feedexport.py | 8 ++++---- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 4f0a946de..83849ca13 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -21,7 +21,7 @@ from zope.interface import Interface, implementer from scrapy import Spider, signals from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.extensions.postprocessing import PostProcessingManager -from scrapy.utils.boto import is_boto3_available, is_botocore_available +from scrapy.utils.boto import is_botocore_available from scrapy.utils.conf import feed_complete_default_values_from_settings from scrapy.utils.ftp import ftp_store_file from scrapy.utils.log import failure_to_exc_info @@ -30,6 +30,13 @@ from scrapy.utils.python import get_func_args, without_none_values logger = logging.getLogger(__name__) +try: + import boto3 # noqa: F401 + + IS_BOTO3_AVAILABLE = True +except ImportError: + IS_BOTO3_AVAILABLE = False + def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs): argument_names = get_func_args(builder) @@ -173,9 +180,8 @@ class S3FeedStorage(BlockingFeedStorage): self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url - self._using_boto3 = is_boto3_available() - if self._using_boto3: + if IS_BOTO3_AVAILABLE: import boto3.session session = boto3.session.Session() @@ -228,7 +234,7 @@ class S3FeedStorage(BlockingFeedStorage): def _store_in_thread(self, file): file.seek(0) - if self._using_boto3: + if IS_BOTO3_AVAILABLE: kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {} self.s3_client.upload_fileobj( Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py index 7b18b6bcf..085ee7d25 100644 --- a/scrapy/utils/boto.py +++ b/scrapy/utils/boto.py @@ -8,12 +8,3 @@ def is_botocore_available(): return True except ImportError: return False - - -def is_boto3_available(): - try: - import boto3 # noqa: F401 - - return True - except ImportError: - return False diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 2e350df65..7df3e6dd3 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -35,6 +35,7 @@ import scrapy from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.exporters import CsvItemExporter, JsonItemExporter from scrapy.extensions.feedexport import ( + IS_BOTO3_AVAILABLE, BlockingFeedStorage, FeedExporter, FileFeedStorage, @@ -44,7 +45,6 @@ from scrapy.extensions.feedexport import ( S3FeedStorage, StdoutFeedStorage, _FeedSlot, - is_boto3_available, ) from scrapy.settings import Settings from scrapy.utils.python import to_unicode @@ -287,7 +287,7 @@ class S3FeedStorageTest(unittest.TestCase): file = mock.MagicMock() - if is_boto3_available(): + if IS_BOTO3_AVAILABLE: storage.s3_client = mock.MagicMock() yield storage.store(file) self.assertEqual( @@ -413,7 +413,7 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - if is_boto3_available(): + if IS_BOTO3_AVAILABLE: acl = ( storage.s3_client.upload_fileobj.call_args[1] .get("ExtraArgs", {}) @@ -434,7 +434,7 @@ class S3FeedStorageTest(unittest.TestCase): storage.s3_client = mock.MagicMock() yield storage.store(BytesIO(b"test file")) - if is_boto3_available(): + if IS_BOTO3_AVAILABLE: acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"] else: acl = storage.s3_client.put_object.call_args[1]["ACL"] From cb67bc17b72a1ae619c89cc26d6f526dd2a26338 Mon Sep 17 00:00:00 2001 From: jazzthief Date: Thu, 16 Mar 2023 17:25:05 +0100 Subject: [PATCH 14/14] Remove `botocore` from docs --- docs/topics/feed-exports.rst | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 8aa3e3be4..5eea6aaf9 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -106,7 +106,7 @@ The storages backends supported out of the box are: - :ref:`topics-feed-storage-stdout` Some storage backends may be unavailable if the required external libraries are -not available. For example, the S3 backend is only available if at least the botocore_ +not available. For example, the S3 backend is only available if the boto3_ library is installed. @@ -193,7 +193,7 @@ The feeds are stored on `Amazon S3`_. - ``s3://aws_key:aws_secret@mybucket/path/to/export.csv`` -- Required external libraries: `boto3`_ >= 1.26.70, will fall back to botocore_ if unavailable +- Required external libraries: `boto3`_ >= 1.20.0 The AWS credentials can be passed as user/password in the URI, or they can be passed through the following settings: @@ -780,6 +780,5 @@ source spider in the feed URI: .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier .. _Amazon S3: https://aws.amazon.com/s3/ .. _boto3: https://github.com/boto/boto3 -.. _botocore: https://github.com/boto/botocore .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl .. _Google Cloud Storage: https://cloud.google.com/storage/