From 467553cc2922d1d844fa136a036716eef3158eb9 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Fri, 5 Feb 2016 13:30:22 +0300 Subject: [PATCH] fix anon test: in this case we do no signing, just change the url --- scrapy/core/downloader/handlers/s3.py | 7 +++++-- tests/test_downloader_handlers.py | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py index d3feb9815..6d28f866e 100644 --- a/scrapy/core/downloader/handlers/s3.py +++ b/scrapy/core/downloader/handlers/s3.py @@ -47,9 +47,10 @@ class S3DownloadHandler(object): # If no credentials could be found anywhere, # consider this an anonymous connection request by default; # unless 'anon' was set explicitly (True/False). - anon = kw.get('anon', None) + anon = kw.get('anon') if anon is None and not aws_access_key_id and not aws_secret_access_key: kw['anon'] = True + self.anon = kw.get('anon') self._signer = None try: @@ -80,7 +81,9 @@ class S3DownloadHandler(object): bucket = p.hostname path = p.path + '?' + p.query if p.query else p.path url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path) - if self._signer is not None: + if self.anon: + request = request.replace(url=url) + elif self._signer is not None: import botocore.awsrequest from botocore.vendored.requests.structures import CaseInsensitiveDict print(url, request.headers) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 06e232503..0f79a208d 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -458,8 +458,10 @@ class S3AnonTestCase(BaseS3TestCase): def test_anon_request(self): req = Request('s3://aws-publicdatasets/') httpreq = self.download_request(req, self.spider) - self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True) - self.assertEqual(self.s3reqh.conn.anon, True) + self.assertEqual(hasattr(self.s3reqh, 'anon'), True) + self.assertEqual(self.s3reqh.anon, True) + self.assertEqual( + httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/') class S3TestCase(unittest.TestCase):