diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index 311815b70..38cfd1e10 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -35,7 +35,7 @@ def get_s3_connection(): class S3DownloadHandler(object): def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \ - httpdownloadhandler=HTTPDownloadHandler): + httpdownloadhandler=HTTPDownloadHandler, **kw): _S3Connection = get_s3_connection() if _S3Connection is None: @@ -46,8 +46,15 @@ class S3DownloadHandler(object): if not aws_secret_access_key: aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY'] + # If no credentials could be found anywhere, + # consider this an anonymous connection request by default; + # unless 'anon' was set explicitly (True/False). + anon = kw.get('anon', None) + if anon is None and not aws_access_key_id and not aws_secret_access_key: + kw['anon'] = True + try: - self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key) + self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key, **kw) except Exception as ex: raise NotConfigured(str(ex)) self._download_http = httpdownloadhandler(settings).download_request diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index e4d957d8e..8280b21aa 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -403,6 +403,23 @@ class HttpDownloadHandlerMock(object): def download_request(self, request, spider): return request +class S3AnonTestCase(unittest.TestCase): + skip = 'boto' not in optional_features and 'missing boto library' + + def setUp(self): + self.s3reqh = S3DownloadHandler(Settings(), + httpdownloadhandler=HttpDownloadHandlerMock, + #anon=True, # is implicit + ) + self.download_request = self.s3reqh.download_request + self.spider = Spider('foo') + + def test_anon_request(self): + req = Request('s3://aws-publicdatasets/') + httpreq = self.download_request(req, self.spider) + self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True) + self.assertEqual(self.s3reqh.conn.anon, True) + class S3TestCase(unittest.TestCase): download_handler_cls = S3DownloadHandler try: @@ -420,8 +437,8 @@ class S3TestCase(unittest.TestCase): AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o' def setUp(self): - s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID, \ - self.AWS_SECRET_ACCESS_KEY, \ + s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID, + self.AWS_SECRET_ACCESS_KEY, httpdownloadhandler=HttpDownloadHandlerMock) self.download_request = s3reqh.download_request self.spider = Spider('foo')