RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5833 from GodelTech/issue_960_s3_feed_export 

Fix #960: S3 Feed Export throws boto error
This commit is contained in:
Andrey Rakhmatullin 2023-06-13 18:44:06 +04:00 committed by GitHub
parent 9e0bfc4a3d cb67bc17b7
commit 85fe88f80f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 157 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
.github/workflows/tests-ubuntu.yml vendored
View File

@ -37,10 +37,19 @@ jobs:
- python-version: pypy3.7
env:
TOXENV: pypy3-pinned
- python-version: 3.7.13
env:
TOXENV: extra-deps-pinned
- python-version: 3.7.13
env:
TOXENV: botocore-pinned
- python-version: "3.11"
env:
TOXENV: extra-deps
- python-version: "3.11"
env:
TOXENV: botocore
steps:
- uses: actions/checkout@v3

8
docs/topics/feed-exports.rst
View File

@ -101,12 +101,12 @@ The storages backends supported out of the box are:
- :ref:`topics-feed-storage-fs`
- :ref:`topics-feed-storage-ftp`
- :ref:`topics-feed-storage-s3` (requires botocore_)
- :ref:`topics-feed-storage-s3` (requires boto3_)
- :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
- :ref:`topics-feed-storage-stdout`
Some storage backends may be unavailable if the required external libraries are
not available. For example, the S3 backend is only available if the botocore_
not available. For example, the S3 backend is only available if the boto3_
library is installed.
@ -199,7 +199,7 @@ The feeds are stored on `Amazon S3`_.
- ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
- Required external libraries: `botocore`_ >= 1.4.87
- Required external libraries: `boto3`_ >= 1.20.0
The AWS credentials can be passed as user/password in the URI, or they can be
passed through the following settings:
@ -799,6 +799,6 @@ source spider in the feed URI:
.. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
.. _Amazon S3: https://aws.amazon.com/s3/
.. _botocore: https://github.com/boto/botocore
.. _boto3: https://github.com/boto/boto3
.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
.. _Google Cloud Storage: https://cloud.google.com/storage/

61
scrapy/extensions/feedexport.py
View File

@ -33,6 +33,13 @@ from scrapy.utils.python import get_func_args, without_none_values
logger = logging.getLogger(__name__)
try:
import boto3 # noqa: F401
IS_BOTO3_AVAILABLE = True
except ImportError:
IS_BOTO3_AVAILABLE = False
def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
argument_names = get_func_args(builder)
@ -176,16 +183,38 @@ class S3FeedStorage(BlockingFeedStorage):
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
self.endpoint_url = endpoint_url
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
)
if IS_BOTO3_AVAILABLE:
import boto3.session
session = boto3.session.Session()
self.s3_client = session.client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
)
else:
warnings.warn(
"`botocore` usage has been deprecated for S3 feed "
"export, please use `boto3` to avoid problems",
category=ScrapyDeprecationWarning,
)
import botocore.session
session = botocore.session.get_session()
self.s3_client = session.create_client(
"s3",
aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url,
)
if feed_options and feed_options.get("overwrite", True) is False:
logger.warning(
"S3 does not support appending to files. To "
@ -208,10 +237,16 @@ class S3FeedStorage(BlockingFeedStorage):
def _store_in_thread(self, file):
file.seek(0)
kwargs = {"ACL": self.acl} if self.acl else {}
self.s3_client.put_object(
Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
)
if IS_BOTO3_AVAILABLE:
kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
self.s3_client.upload_fileobj(
Bucket=self.bucketname, Key=self.keyname, Fileobj=file, **kwargs
)
else:
kwargs = {"ACL": self.acl} if self.acl else {}
self.s3_client.put_object(
Bucket=self.bucketname, Key=self.keyname, Body=file, **kwargs
)
file.close()

95
tests/test_feedexport.py
View File

@ -36,6 +36,7 @@ from scrapy import signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.exporters import CsvItemExporter, JsonItemExporter
from scrapy.extensions.feedexport import (
IS_BOTO3_AVAILABLE,
BlockingFeedStorage,
FeedExporter,
FeedSlot,
@ -236,8 +237,10 @@ class BlockingFeedStorageTest(unittest.TestCase):
class S3FeedStorageTest(unittest.TestCase):
def test_parse_credentials(self):
def setUp(self):
skip_if_no_boto()
def test_parse_credentials(self):
aws_credentials = {
"AWS_ACCESS_KEY_ID": "settings_key",
"AWS_SECRET_ACCESS_KEY": "settings_secret",
@ -273,8 +276,6 @@ class S3FeedStorageTest(unittest.TestCase):
@defer.inlineCallbacks
def test_store(self):
skip_if_no_boto()
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
"AWS_SECRET_ACCESS_KEY": "secret_key",
@ -286,30 +287,39 @@ class S3FeedStorageTest(unittest.TestCase):
verifyObject(IFeedStorage, storage)
file = mock.MagicMock()
from botocore.stub import Stubber
with Stubber(storage.s3_client) as stub:
stub.add_response(
"put_object",
expected_params={
"Body": file,
"Bucket": bucket,
"Key": key,
},
service_response={},
)
if IS_BOTO3_AVAILABLE:
storage.s3_client = mock.MagicMock()
yield storage.store(file)
stub.assert_no_pending_responses()
self.assertEqual(
file.method_calls,
[
mock.call.seek(0),
# The call to read does not happen with Stubber
mock.call.close(),
],
storage.s3_client.upload_fileobj.call_args,
mock.call(Bucket=bucket, Key=key, Fileobj=file),
)
else:
from botocore.stub import Stubber
with Stubber(storage.s3_client) as stub:
stub.add_response(
"put_object",
expected_params={
"Body": file,
"Bucket": bucket,
"Key": key,
},
service_response={},
)
yield storage.store(file)
stub.assert_no_pending_responses()
self.assertEqual(
file.method_calls,
[
mock.call.seek(0),
# The call to read does not happen with Stubber
mock.call.close(),
],
)
def test_init_without_acl(self):
storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")
@ -392,8 +402,7 @@ class S3FeedStorageTest(unittest.TestCase):
self.assertEqual(storage.endpoint_url, "https://example.com")
@defer.inlineCallbacks
def test_store_botocore_without_acl(self):
skip_if_no_boto()
def test_store_without_acl(self):
storage = S3FeedStorage(
"s3://mybucket/export.csv",
"access_key",
@ -405,11 +414,18 @@ class S3FeedStorageTest(unittest.TestCase):
storage.s3_client = mock.MagicMock()
yield storage.store(BytesIO(b"test file"))
self.assertNotIn("ACL", storage.s3_client.put_object.call_args[1])
if IS_BOTO3_AVAILABLE:
acl = (
storage.s3_client.upload_fileobj.call_args[1]
.get("ExtraArgs", {})
.get("ACL")
)
else:
acl = storage.s3_client.put_object.call_args[1].get("ACL")
self.assertIsNone(acl)
@defer.inlineCallbacks
def test_store_botocore_with_acl(self):
skip_if_no_boto()
def test_store_with_acl(self):
storage = S3FeedStorage(
"s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
)
@ -419,9 +435,11 @@ class S3FeedStorageTest(unittest.TestCase):
storage.s3_client = mock.MagicMock()
yield storage.store(BytesIO(b"test file"))
self.assertEqual(
storage.s3_client.put_object.call_args[1].get("ACL"), "custom-acl"
)
if IS_BOTO3_AVAILABLE:
acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
else:
acl = storage.s3_client.put_object.call_args[1]["ACL"]
self.assertEqual(acl, "custom-acl")
def test_overwrite_default(self):
with LogCapture() as log:
@ -889,15 +907,10 @@ class FeedExportTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_stats_multiple_file(self):
settings = {
"AWS_ACCESS_KEY_ID": "access_key",
"AWS_SECRET_ACCESS_KEY": "secret_key",
"FEEDS": {
printf_escape(path_to_url(str(self._random_temp_filename()))): {
"format": "json",
},
"s3://bucket/key/foo.csv": {
"format": "csv",
},
"stdout:": {
"format": "xml",
},
@ -909,18 +922,12 @@ class FeedExportTest(FeedExportTestBase):
self.assertIn(
"feedexport/success_count/FileFeedStorage", crawler.stats.get_stats()
)
self.assertIn(
"feedexport/success_count/S3FeedStorage", crawler.stats.get_stats()
)
self.assertIn(
"feedexport/success_count/StdoutFeedStorage", crawler.stats.get_stats()
)
self.assertEqual(
crawler.stats.get_value("feedexport/success_count/FileFeedStorage"), 1
)
self.assertEqual(
crawler.stats.get_value("feedexport/success_count/S3FeedStorage"), 1
)
self.assertEqual(
crawler.stats.get_value("feedexport/success_count/StdoutFeedStorage"), 1
)
@ -2587,7 +2594,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_s3_export(self):
skip_if_no_boto()
bucket = "mybucket"
items = [
self.MyItem({"foo": "bar1", "egg": "spam1"}),
@ -2836,6 +2842,9 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
maxDiff = None
def setUp(self):
skip_if_no_boto()
def test_init(self):
settings_dict = {
"FEED_URI": "file:///tmp/foobar",

12
tests/test_pipeline_files.py
View File

@ -225,12 +225,16 @@ class FilesPipelineTestCase(unittest.TestCase):
class FilesPipelineTestCaseFieldsMixin:
def setUp(self):
self.tempdir = mkdtemp()
def tearDown(self):
rmtree(self.tempdir)
def test_item_fields_default(self):
url = "http://www.example.com/files/1.txt"
item = self.item_class(name="item1", file_urls=[url])
pipeline = FilesPipeline.from_settings(
Settings({"FILES_STORE": "s3://example/files/"})
)
pipeline = FilesPipeline.from_settings(Settings({"FILES_STORE": self.tempdir}))
requests = list(pipeline.get_media_requests(item, None))
self.assertEqual(requests[0].url, url)
results = [(True, {"url": url})]
@ -245,7 +249,7 @@ class FilesPipelineTestCaseFieldsMixin:
pipeline = FilesPipeline.from_settings(
Settings(
{
"FILES_STORE": "s3://example/files/",
"FILES_STORE": self.tempdir,
"FILES_URLS_FIELD": "custom_file_urls",
"FILES_RESULT_FIELD": "custom_files",
}

46
tox.ini
View File

@ -18,8 +18,6 @@ deps =
mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
# newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy'
# Extras
botocore>=1.4.87
passenv =
S3_TEST_FILE_URI
AWS_ACCESS_KEY_ID
@ -90,11 +88,6 @@ deps =
# mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
# above, hence we do not install it in pinned environments at the moment
# Extras
botocore==1.4.87
google-cloud-storage==1.29.0
Pillow==7.1.0
setenv =
_SCRAPY_PINNED=true
install_command =
@ -126,14 +119,26 @@ commands = {[pinned]commands}
basepython = python3
deps =
{[testenv]deps}
boto
boto3
google-cloud-storage
# Twisted[http2] currently forces old mitmproxy because of h2 version
# restrictions in their deps, so we need to pin old markupsafe here too.
markupsafe < 2.1.0
robotexclusionrulesparser
Pillow>=4.0.0
Twisted[http2]>=17.9.0
Pillow
Twisted[http2]
[testenv:extra-deps-pinned]
basepython = python3.7
deps =
{[pinned]deps}
boto3==1.20.0
google-cloud-storage==1.29.0
Pillow==7.1.0
robotexclusionrulesparser==1.6.2
install_command = {[pinned]install_command}
setenv =
{[pinned]setenv}
[testenv:asyncio]
commands =
@ -193,3 +198,24 @@ deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
# Run S3 tests with botocore installed but without boto3.
[testenv:botocore]
deps =
{[testenv]deps}
botocore>=1.4.87
commands =
pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
[testenv:botocore-pinned]
basepython = python3.7
deps =
{[pinned]deps}
botocore==1.4.87
install_command = {[pinned]install_command}
setenv =
{[pinned]setenv}
commands =
pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}