RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #5210 from laggardkernel/feature/aws-session-token

This commit is contained in:
Andrey Rahmatullin 2021-07-30 15:15:55 +05:00 committed by GitHub
parent cc89f6be38 8e7b96d8a2
commit b406affd5b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 39 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
docs/topics/feed-exports.rst
View File

@ -135,7 +135,7 @@ Here are some examples to illustrate:
- ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json``
.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
.. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
they can also be used as storage URI parameters.
@ -200,6 +200,9 @@ passed through the following settings:
- :setting:`AWS_ACCESS_KEY_ID`
- :setting:`AWS_SECRET_ACCESS_KEY`
- :setting:`AWS_SESSION_TOKEN` (only needed for `temporary security credentials`_)
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
You can also define a custom ACL and custom endpoint for exported feeds using this setting:
@ -357,7 +360,7 @@ For instance::
'item_export_kwargs': {
'export_empty_fields': True,
},
},
},
'/home/user/documents/items.xml': {
'format': 'xml',
'fields': ['name', 'price'],

13
docs/topics/settings.rst
View File

@ -204,6 +204,19 @@ Default: ``None``
The AWS secret key used by code that requires access to `Amazon Web services`_,
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
.. setting:: AWS_SESSION_TOKEN
AWS_SESSION_TOKEN
-----------------
Default: ``None``
The AWS security token used by code that requires access to `Amazon Web services`_,
such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using
`temporary security credentials`_.
.. _temporary security credentials: https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#temporary-access-keys
.. setting:: AWS_ENDPOINT_URL
AWS_ENDPOINT_URL

5
scrapy/core/downloader/handlers/s3.py
View File

@ -10,6 +10,7 @@ class S3DownloadHandler:
def __init__(self, settings, *,
crawler=None,
aws_access_key_id=None, aws_secret_access_key=None,
aws_session_token=None,
httpdownloadhandler=HTTPDownloadHandler, **kw):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
@ -18,6 +19,8 @@ class S3DownloadHandler:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
if not aws_secret_access_key:
aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
if not aws_session_token:
aws_session_token = settings['AWS_SESSION_TOKEN']
# If no credentials could be found anywhere,
# consider this an anonymous connection request by default;
@ -36,7 +39,7 @@ class S3DownloadHandler:
if not self.anon:
SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3']
self._signer = SignerCls(botocore.credentials.Credentials(
aws_access_key_id, aws_secret_access_key))
aws_access_key_id, aws_secret_access_key, aws_session_token))
_http_handler = create_instance(
objcls=httpdownloadhandler,

5
scrapy/extensions/feedexport.py
View File

@ -154,13 +154,14 @@ class FileFeedStorage:
class S3FeedStorage(BlockingFeedStorage):
def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *,
feed_options=None):
feed_options=None, session_token=None):
if not is_botocore_available():
raise NotConfigured('missing botocore library')
u = urlparse(uri)
self.bucketname = u.hostname
self.access_key = u.username or access_key
self.secret_key = u.password or secret_key
self.session_token = session_token
self.keyname = u.path[1:] # remove first "/"
self.acl = acl
self.endpoint_url = endpoint_url
@ -169,6 +170,7 @@ class S3FeedStorage(BlockingFeedStorage):
self.s3_client = session.create_client(
's3', aws_access_key_id=self.access_key,
aws_secret_access_key=self.secret_key,
aws_session_token=self.session_token,
endpoint_url=self.endpoint_url)
if feed_options and feed_options.get('overwrite', True) is False:
logger.warning('S3 does not support appending to files. To '
@ -182,6 +184,7 @@ class S3FeedStorage(BlockingFeedStorage):
uri,
access_key=crawler.settings['AWS_ACCESS_KEY_ID'],
secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'],
session_token=crawler.settings['AWS_SESSION_TOKEN'],
acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None,
endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None,
feed_options=feed_options,

3
scrapy/pipelines/files.py
View File

@ -79,6 +79,7 @@ class FSFilesStore:
class S3FilesStore:
AWS_ACCESS_KEY_ID = None
AWS_SECRET_ACCESS_KEY = None
AWS_SESSION_TOKEN = None
AWS_ENDPOINT_URL = None
AWS_REGION_NAME = None
AWS_USE_SSL = None
@ -98,6 +99,7 @@ class S3FilesStore:
's3',
aws_access_key_id=self.AWS_ACCESS_KEY_ID,
aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY,
aws_session_token=self.AWS_SESSION_TOKEN,
endpoint_url=self.AWS_ENDPOINT_URL,
region_name=self.AWS_REGION_NAME,
use_ssl=self.AWS_USE_SSL,
@ -349,6 +351,7 @@ class FilesPipeline(MediaPipeline):
s3store = cls.STORE_SCHEMES['s3']
s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN']
s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']
s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']
s3store.AWS_USE_SSL = settings['AWS_USE_SSL']

1
scrapy/pipelines/images.py
View File

@ -92,6 +92,7 @@ class ImagesPipeline(FilesPipeline):
s3store = cls.STORE_SCHEMES['s3']
s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN']
s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL']
s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME']
s3store.AWS_USE_SSL = settings['AWS_USE_SSL']

12
tests/test_feedexport.py
View File

@ -244,7 +244,8 @@ class S3FeedStorageTest(unittest.TestCase):
def test_parse_credentials(self):
skip_if_no_boto()
aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key',
'AWS_SECRET_ACCESS_KEY': 'settings_secret'}
'AWS_SECRET_ACCESS_KEY': 'settings_secret',
'AWS_SESSION_TOKEN': 'settings_token'}
crawler = get_crawler(settings_dict=aws_credentials)
# Instantiate with crawler
storage = S3FeedStorage.from_crawler(
@ -253,12 +254,15 @@ class S3FeedStorageTest(unittest.TestCase):
)
self.assertEqual(storage.access_key, 'settings_key')
self.assertEqual(storage.secret_key, 'settings_secret')
self.assertEqual(storage.session_token, 'settings_token')
# Instantiate directly
storage = S3FeedStorage('s3://mybucket/export.csv',
aws_credentials['AWS_ACCESS_KEY_ID'],
aws_credentials['AWS_SECRET_ACCESS_KEY'])
aws_credentials['AWS_SECRET_ACCESS_KEY'],
session_token=aws_credentials['AWS_SESSION_TOKEN'])
self.assertEqual(storage.access_key, 'settings_key')
self.assertEqual(storage.secret_key, 'settings_secret')
self.assertEqual(storage.session_token, 'settings_token')
# URI priority > settings priority
storage = S3FeedStorage('s3://uri_key:uri_secret@mybucket/export.csv',
aws_credentials['AWS_ACCESS_KEY_ID'],
@ -1957,8 +1961,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
def __init__(self, uri, access_key, secret_key, acl, endpoint_url):
super().__init__(uri, access_key, secret_key, acl, endpoint_url)
def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs):
super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs)
class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage):

1
tox.ini
View File

@ -23,6 +23,7 @@ passenv =
S3_TEST_FILE_URI
AWS_ACCESS_KEY_ID
AWS_SECRET_ACCESS_KEY
AWS_SESSION_TOKEN
GCS_TEST_FILE_URI
GCS_PROJECT_ID
#allow tox virtualenv to upgrade pip/wheel/setuptools