Support anonymous connections in S3DownloadHandler

Also consider any unknown keyword args for S3DownloadHandler as
arguments to pass on to S3Connection (e.g. proxy settings).

scrapy/core/downloader/handlers/s3.py

@@ -35,7 +35,7 @@ def get_s3_connection():
 class S3DownloadHandler(object):
 
     def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \
-            httpdownloadhandler=HTTPDownloadHandler):
+            httpdownloadhandler=HTTPDownloadHandler, **kw):
 
         _S3Connection = get_s3_connection()
         if _S3Connection is None:
@@ -46,8 +46,15 @@ class S3DownloadHandler(object):
         if not aws_secret_access_key:
             aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
 
+        # If no credentials could be found anywhere,
+        # consider this an anonymous connection request by default;
+        # unless 'anon' was set explicitly (True/False).
+        anon = kw.get('anon', None)
+        if anon is None and not aws_access_key_id and not aws_secret_access_key:
+            kw['anon'] = True
+
         try:
-            self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key)
+            self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key, **kw)
         except Exception as ex:
             raise NotConfigured(str(ex))
         self._download_http = httpdownloadhandler(settings).download_request

tests/test_downloader_handlers.py

@@ -403,6 +403,23 @@ class HttpDownloadHandlerMock(object):
     def download_request(self, request, spider):
         return request
 
+class S3AnonTestCase(unittest.TestCase):
+    skip = 'boto' not in optional_features and 'missing boto library'
+
+    def setUp(self):
+        self.s3reqh = S3DownloadHandler(Settings(),
+                httpdownloadhandler=HttpDownloadHandlerMock,
+                #anon=True, # is implicit
+        )
+        self.download_request = self.s3reqh.download_request
+        self.spider = Spider('foo')
+
+    def test_anon_request(self):
+        req = Request('s3://aws-publicdatasets/')
+        httpreq = self.download_request(req, self.spider)
+        self.assertEqual(hasattr(self.s3reqh.conn, 'anon'), True)
+        self.assertEqual(self.s3reqh.conn.anon, True)
+
 class S3TestCase(unittest.TestCase):
     download_handler_cls = S3DownloadHandler
     try:
@@ -420,8 +437,8 @@ class S3TestCase(unittest.TestCase):
     AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
 
     def setUp(self):
-        s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID, \
-                self.AWS_SECRET_ACCESS_KEY, \
+        s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID,
+                self.AWS_SECRET_ACCESS_KEY,
                 httpdownloadhandler=HttpDownloadHandlerMock)
         self.download_request = s3reqh.download_request
         self.spider = Spider('foo')