fix anon test: in this case we do no signing, just change the url

scrapy/core/downloader/handlers/s3.py

@@ -47,9 +47,10 @@ class S3DownloadHandler(object):
         # If no credentials could be found anywhere,
         # consider this an anonymous connection request by default;
         # unless 'anon' was set explicitly (True/False).
-        anon = kw.get('anon', None)
+        anon = kw.get('anon')
         if anon is None and not aws_access_key_id and not aws_secret_access_key:
             kw['anon'] = True
+        self.anon = kw.get('anon')
 
         self._signer = None
         try:
@@ -80,7 +81,9 @@ class S3DownloadHandler(object):
         bucket = p.hostname
         path = p.path + '?' + p.query if p.query else p.path
         url = '%s://%s.s3.amazonaws.com%s' % (scheme, bucket, path)
-        if self._signer is not None:
+        if self.anon:
+            request = request.replace(url=url)
+        elif self._signer is not None:
             import botocore.awsrequest
             from botocore.vendored.requests.structures import CaseInsensitiveDict
             print(url, request.headers)

tests/test_downloader_handlers.py

@@ -458,8 +458,10 @@ class S3AnonTestCase(BaseS3TestCase):
     def test_anon_request(self):
         req = Request('s3://aws-publicdatasets/')
         httpreq = self.download_request(req, self.spider)
-        self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True)
-        self.assertEqual(self.s3reqh.conn.anon, True)
+        self.assertEqual(hasattr(self.s3reqh, 'anon'), True)
+        self.assertEqual(self.s3reqh.anon, True)
+        self.assertEqual(
+            httpreq.url, 'http://aws-publicdatasets.s3.amazonaws.com/')
 
 
 class S3TestCase(unittest.TestCase):