From 7e4321f201a795166d779f2aa0b36d38cb50106e Mon Sep 17 00:00:00 2001 From: laggardkernel Date: Mon, 19 Jul 2021 12:00:42 +0800 Subject: [PATCH] Add support for temporary security credential in AWS auth --- docs/topics/feed-exports.rst | 5 +++-- docs/topics/settings.rst | 14 ++++++++++++++ scrapy/core/downloader/handlers/s3.py | 5 ++++- scrapy/extensions/feedexport.py | 5 ++++- scrapy/pipelines/files.py | 3 +++ scrapy/pipelines/images.py | 1 + tests/test_feedexport.py | 12 ++++++++---- tox.ini | 1 + 8 files changed, 38 insertions(+), 8 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 216a8bc52..af60de716 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -135,7 +135,7 @@ Here are some examples to illustrate: - ``s3://mybucket/scraping/feeds/%(name)s/%(time)s.json`` -.. note:: :ref:`Spider arguments ` become spider attributes, hence +.. note:: :ref:`Spider arguments ` become spider attributes, hence they can also be used as storage URI parameters. @@ -200,6 +200,7 @@ passed through the following settings: - :setting:`AWS_ACCESS_KEY_ID` - :setting:`AWS_SECRET_ACCESS_KEY` +- :setting:`AWS_SESSION_TOKEN` (Optional) You can also define a custom ACL and custom endpoint for exported feeds using this setting: @@ -357,7 +358,7 @@ For instance:: 'item_export_kwargs': { 'export_empty_fields': True, }, - }, + }, '/home/user/documents/items.xml': { 'format': 'xml', 'fields': ['name', 'price'], diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 1290b4a5e..58daafa6f 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -204,6 +204,20 @@ Default: ``None`` The AWS secret key used by code that requires access to `Amazon Web services`_, such as the :ref:`S3 feed storage backend `. +.. setting:: AWS_SESSION_TOKEN + +AWS_SESSION_TOKEN +----------------- + +Default: ``None`` (Optional) + +The AWS security token used by code that requires access to `Amazon Web services`_, +such as the :ref:`S3 feed storage backend `. + +The security token is only required by a *temporary security credentials*. +Using of temporary security credentials is discouraged cause the credentials +are short term. It may expires before the scraping is done. + .. setting:: AWS_ENDPOINT_URL AWS_ENDPOINT_URL diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 31f1be31a..51ca1ed5e 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -10,6 +10,7 @@ class S3DownloadHandler: def __init__(self, settings, *, crawler=None, aws_access_key_id=None, aws_secret_access_key=None, + aws_session_token=None, httpdownloadhandler=HTTPDownloadHandler, **kw): if not is_botocore_available(): raise NotConfigured('missing botocore library') @@ -18,6 +19,8 @@ class S3DownloadHandler: aws_access_key_id = settings['AWS_ACCESS_KEY_ID'] if not aws_secret_access_key: aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY'] + if not aws_session_token: + aws_session_token = settings['AWS_SESSION_TOKEN'] # If no credentials could be found anywhere, # consider this an anonymous connection request by default; @@ -36,7 +39,7 @@ class S3DownloadHandler: if not self.anon: SignerCls = botocore.auth.AUTH_TYPE_MAPS['s3'] self._signer = SignerCls(botocore.credentials.Credentials( - aws_access_key_id, aws_secret_access_key)) + aws_access_key_id, aws_secret_access_key, aws_session_token)) _http_handler = create_instance( objcls=httpdownloadhandler, diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index bd4808e2b..564c736f2 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -154,13 +154,14 @@ class FileFeedStorage: class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None, acl=None, endpoint_url=None, *, - feed_options=None): + feed_options=None, session_token=None): if not is_botocore_available(): raise NotConfigured('missing botocore library') u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key self.secret_key = u.password or secret_key + self.session_token = session_token self.keyname = u.path[1:] # remove first "/" self.acl = acl self.endpoint_url = endpoint_url @@ -169,6 +170,7 @@ class S3FeedStorage(BlockingFeedStorage): self.s3_client = session.create_client( 's3', aws_access_key_id=self.access_key, aws_secret_access_key=self.secret_key, + aws_session_token=self.session_token, endpoint_url=self.endpoint_url) if feed_options and feed_options.get('overwrite', True) is False: logger.warning('S3 does not support appending to files. To ' @@ -182,6 +184,7 @@ class S3FeedStorage(BlockingFeedStorage): uri, access_key=crawler.settings['AWS_ACCESS_KEY_ID'], secret_key=crawler.settings['AWS_SECRET_ACCESS_KEY'], + session_token=crawler.settings['AWS_SESSION_TOKEN'], acl=crawler.settings['FEED_STORAGE_S3_ACL'] or None, endpoint_url=crawler.settings['AWS_ENDPOINT_URL'] or None, feed_options=feed_options, diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py index 13ecd4e6c..8766ef66f 100644 --- a/scrapy/pipelines/files.py +++ b/scrapy/pipelines/files.py @@ -79,6 +79,7 @@ class FSFilesStore: class S3FilesStore: AWS_ACCESS_KEY_ID = None AWS_SECRET_ACCESS_KEY = None + AWS_SESSION_TOKEN = None AWS_ENDPOINT_URL = None AWS_REGION_NAME = None AWS_USE_SSL = None @@ -98,6 +99,7 @@ class S3FilesStore: 's3', aws_access_key_id=self.AWS_ACCESS_KEY_ID, aws_secret_access_key=self.AWS_SECRET_ACCESS_KEY, + aws_session_token=self.AWS_SESSION_TOKEN, endpoint_url=self.AWS_ENDPOINT_URL, region_name=self.AWS_REGION_NAME, use_ssl=self.AWS_USE_SSL, @@ -349,6 +351,7 @@ class FilesPipeline(MediaPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN'] s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py index e3ab23ea5..9c99dc69e 100644 --- a/scrapy/pipelines/images.py +++ b/scrapy/pipelines/images.py @@ -92,6 +92,7 @@ class ImagesPipeline(FilesPipeline): s3store = cls.STORE_SCHEMES['s3'] s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID'] s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY'] + s3store.AWS_SESSION_TOKEN = settings['AWS_SESSION_TOKEN'] s3store.AWS_ENDPOINT_URL = settings['AWS_ENDPOINT_URL'] s3store.AWS_REGION_NAME = settings['AWS_REGION_NAME'] s3store.AWS_USE_SSL = settings['AWS_USE_SSL'] diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index da0b2c786..389808306 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -244,7 +244,8 @@ class S3FeedStorageTest(unittest.TestCase): def test_parse_credentials(self): skip_if_no_boto() aws_credentials = {'AWS_ACCESS_KEY_ID': 'settings_key', - 'AWS_SECRET_ACCESS_KEY': 'settings_secret'} + 'AWS_SECRET_ACCESS_KEY': 'settings_secret', + 'AWS_SESSION_TOKEN': 'settings_token'} crawler = get_crawler(settings_dict=aws_credentials) # Instantiate with crawler storage = S3FeedStorage.from_crawler( @@ -253,12 +254,15 @@ class S3FeedStorageTest(unittest.TestCase): ) self.assertEqual(storage.access_key, 'settings_key') self.assertEqual(storage.secret_key, 'settings_secret') + self.assertEqual(storage.session_token, 'settings_token') # Instantiate directly storage = S3FeedStorage('s3://mybucket/export.csv', aws_credentials['AWS_ACCESS_KEY_ID'], - aws_credentials['AWS_SECRET_ACCESS_KEY']) + aws_credentials['AWS_SECRET_ACCESS_KEY'], + session_token=aws_credentials['AWS_SESSION_TOKEN']) self.assertEqual(storage.access_key, 'settings_key') self.assertEqual(storage.secret_key, 'settings_secret') + self.assertEqual(storage.session_token, 'settings_token') # URI priority > settings priority storage = S3FeedStorage('s3://uri_key:uri_secret@mybucket/export.csv', aws_credentials['AWS_ACCESS_KEY_ID'], @@ -1957,8 +1961,8 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase): class S3FeedStorageWithoutFeedOptions(S3FeedStorage): - def __init__(self, uri, access_key, secret_key, acl, endpoint_url): - super().__init__(uri, access_key, secret_key, acl, endpoint_url) + def __init__(self, uri, access_key, secret_key, acl, endpoint_url, **kwargs): + super().__init__(uri, access_key, secret_key, acl, endpoint_url, **kwargs) class S3FeedStorageWithoutFeedOptionsWithFromCrawler(S3FeedStorage): diff --git a/tox.ini b/tox.ini index 4c4bbff6e..96050223b 100644 --- a/tox.ini +++ b/tox.ini @@ -23,6 +23,7 @@ passenv = S3_TEST_FILE_URI AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY + AWS_SESSION_TOKEN GCS_TEST_FILE_URI GCS_PROJECT_ID #allow tox virtualenv to upgrade pip/wheel/setuptools