From 98d74d1083be6afa7553a5950f89fc5ac446272f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Fri, 20 Jul 2018 12:08:49 -0300 Subject: [PATCH] Requested changes --- scrapy/extensions/feedexport.py | 29 +++++++++++++++-------------- scrapy/utils/misc.py | 1 + tests/test_feedexport.py | 4 +++- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 067887d94..7c7db387e 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -96,21 +96,22 @@ class S3FeedStorage(BlockingFeedStorage): def __init__(self, uri, access_key=None, secret_key=None): # BEGIN Backwards compatibility for initialising without keys (and # without using from_crawler) - from scrapy.conf import settings no_defaults = access_key is None and secret_key is None - if no_defaults and ('AWS_ACCESS_KEY_ID' in settings or - 'AWS_SECRET_ACCESS_KEY' in settings): - import warnings - from scrapy.exceptions import ScrapyDeprecationWarning - warnings.warn( - "Initialising `scrapy.extensions.feedexport.S3FeedStorage` " - "without AWS keys is deprecated. Please supply credentials or " - "use the `from_crawler()` constructor.", - category=ScrapyDeprecationWarning, - stacklevel=2 - ) - access_key = settings['AWS_ACCESS_KEY_ID'] - secret_key = settings['AWS_SECRET_ACCESS_KEY'] + if no_defaults: + from scrapy.conf import settings + if 'AWS_ACCESS_KEY_ID' in settings or 'AWS_SECRET_ACCESS_KEY' in settings: + import warnings + from scrapy.exceptions import ScrapyDeprecationWarning + warnings.warn( + "Initialising `scrapy.extensions.feedexport.S3FeedStorage` " + "without AWS keys is deprecated. Please supply credentials or " + "use the `from_crawler()` constructor.", + category=ScrapyDeprecationWarning, + stacklevel=2 + ) + access_key = settings['AWS_ACCESS_KEY_ID'] + secret_key = settings['AWS_SECRET_ACCESS_KEY'] + # END Backwards compatibility u = urlparse(uri) self.bucketname = u.hostname self.access_key = u.username or access_key diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index 8eb1aabb5..5ccfdcd72 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -118,6 +118,7 @@ def rel_has_nofollow(rel): """Return True if link rel attribute has nofollow type""" return True if rel is not None and 'nofollow' in rel.split() else False + def create_instance(objcls, settings, crawler, *args, **kwargs): """Construct a class instance using its ``from_crawler`` or ``from_settings`` constructors, if available. diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index eeb1bc2a4..380ed971b 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -173,7 +173,9 @@ class S3FeedStorageTest(unittest.TestCase): uri = os.environ.get('S3_TEST_FILE_URI') if not uri: raise unittest.SkipTest("No S3 URI available for testing") - storage = S3FeedStorage(uri, Settings()) + access_key = os.environ.get('AWS_ACCESS_KEY_ID') + secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') + storage = S3FeedStorage(uri, access_key, secret_key) verifyObject(IFeedStorage, storage) file = storage.open(scrapy.Spider("default")) expected_content = b"content: \xe2\x98\x83"