diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index d3feb9815..6d28f866e 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -47,9 +47,10 @@ class S3DownloadHandler(object): # If no credentials could be found anywhere, # consider this an anonymous connection request by default; # unless 'anon' was set explicitly (True/False). - anon = kw.get('anon', None) + anon = kw.get('anon') if anon is None and not aws_access_key_id and not aws_secret_access_key: kw['anon'] = True + self.anon = kw.get('anon') self._signer = None try: @@ -80,7 +81,9 @@ class S3DownloadHandler(object): bucket = p.hostname path = p.path + '?' + p.query if p.query else p.path url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path) - if self._signer is not None: + if self.anon: + request = request.replace(url=url) + elif self._signer is not None: import botocore.awsrequest from botocore.vendored.requests.structures import CaseInsensitiveDict print(url, request.headers) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 06e232503..0f79a208d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -458,8 +458,10 @@ class S3AnonTestCase(BaseS3TestCase): def test_anon_request(self): req = Request('s3://aws-publicdatasets/') httpreq = self.download_request(req, self.spider) - self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True) - self.assertEqual(self.s3reqh.conn.anon, True) + self.assertEqual(hasattr(self.s3reqh, 'anon'), True) + self.assertEqual(self.s3reqh.anon, True) + self.assertEqual( + httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/') class S3TestCase(unittest.TestCase):