Add S3 scheme request handler. closes #222

scrapy/__init__.py

@@ -26,3 +26,10 @@ except ImportError:
     pass
 else:
     optional_features.add('ssl')
+
+try:
+    import boto
+except ImportError:
+    pass
+else:
+    optional_features.add('boto')

scrapy/conf/default_settings.py

@@ -188,6 +188,7 @@ REQUEST_HANDLERS_BASE = {
     'file': 'scrapy.core.downloader.handlers.file.FileRequestHandler',
     'http': 'scrapy.core.downloader.handlers.http.HttpRequestHandler',
     'https': 'scrapy.core.downloader.handlers.http.HttpRequestHandler',
+    's3': 'scrapy.core.downloader.handlers.s3.S3RequestHandler',
 }
 
 REQUESTS_QUEUE_SIZE = 0

scrapy/core/downloader/handlers/s3.py

@@ -0,0 +1,34 @@
+from scrapy import optional_features
+from scrapy.exceptions import NotConfigured
+from scrapy.utils.httpobj import urlparse_cached
+from scrapy.conf import settings
+from .http import HttpRequestHandler
+
+
+class S3RequestHandler(object):
+
+    def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, \
+            httprequesthandler=HttpRequestHandler):
+        if 'boto' not in optional_features:
+            raise NotConfigured("missing boto library")
+
+        if not aws_access_key_id:
+            aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
+        if not aws_secret_access_key:
+            aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
+
+        from boto import connect_s3
+        try:
+            self.conn = connect_s3(aws_access_key_id, aws_secret_access_key)
+        except Exception, ex:
+            raise NotConfigured(str(ex))
+        self._download_http = httprequesthandler().download_request
+
+    def download_request(self, request, spider):
+        p = urlparse_cached(request)
+        scheme = 'https' if request.meta.get('is_secure') else 'http'
+        url = '%s://%s.s3.amazonaws.com%s' % (scheme, p.hostname, p.path)
+        httpreq = request.replace(url=url)
+        self.conn.add_aws_auth_header(httpreq.headers, httpreq.method, \
+                '%s/%s' % (p.hostname, p.path))
+        return self._download_http(httpreq, spider)

scrapy/tests/test_downloader_handlers.py

@@ -12,8 +12,10 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
 from scrapy.core.downloader.webclient import PartialDownloadError
 from scrapy.core.downloader.handlers.file import FileRequestHandler
 from scrapy.core.downloader.handlers.http import HttpRequestHandler
+from scrapy.core.downloader.handlers.s3 import S3RequestHandler
 from scrapy.spider import BaseSpider
 from scrapy.http import Request
+from scrapy import optional_features
 
 
 class FileTestCase(unittest.TestCase):
@@ -179,3 +181,96 @@ class HttpProxyTestCase(unittest.TestCase):
 
         request = Request(self.getURL('path/to/resource'))
         return self.download_request(request, BaseSpider('foo')).addCallback(_test)
+
+
+class HttpRequestHandlerMock(object):
+    def download_request(self, request, spider):
+        return request
+
+class S3TestCase(unittest.TestCase):
+    skip = 'boto' not in optional_features and 'missing boto library'
+
+    # test use same example keys than amazon developer guide
+    # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf
+    # and the tests described here are the examples from that manual
+
+    AWS_ACCESS_KEY_ID = '0PN5J17HBGZHT7JJ3X82'
+    AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
+
+    def setUp(self):
+        s3reqh = S3RequestHandler(self.AWS_ACCESS_KEY_ID, \
+                self.AWS_SECRET_ACCESS_KEY, \
+                httprequesthandler=HttpRequestHandlerMock)
+        self.download_request = s3reqh.download_request
+        self.spider = BaseSpider('foo')
+
+    def test_request_signing1(self):
+        # gets an object from the johnsmith bucket.
+        req = Request('s3://johnsmith/photos/puppy.jpg',
+                headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'})
+        httpreq = self.download_request(req, self.spider)
+        self.assertEqual(httpreq.headers['Authorization'], \
+                'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
+
+    def test_request_signing2(self):
+        # puts an object into the johnsmith bucket.
+        req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
+            'Content-Type': 'image/jpeg',
+            'Date': 'Tue, 27 Mar 2007 21:15:45 +0000',
+            'Content-Length': '94328',
+            })
+        httpreq = self.download_request(req, self.spider)
+        self.assertEqual(httpreq.headers['Authorization'], \
+                'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
+
+    def test_request_signing3(self):
+        # lists the content of the johnsmith bucket.
+        req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \
+                method='GET', headers={
+                    'User-Agent': 'Mozilla/5.0',
+                    'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
+                    })
+        httpreq = self.download_request(req, self.spider)
+        self.assertEqual(httpreq.headers['Authorization'], \
+                'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
+
+    def test_request_signing4(self):
+        # fetches the access control policy sub-resource for the 'johnsmith' bucket.
+        req = Request('s3://johnsmith/?acl', \
+                method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'})
+        httpreq = self.download_request(req, self.spider)
+        self.assertEqual(httpreq.headers['Authorization'], \
+                'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
+
+    def test_request_signing5(self):
+        # deletes an object from the 'johnsmith' bucket using the 
+        # path-style and Date alternative.
+        req = Request('s3://johnsmith/photos/puppy.jpg', \
+                method='DELETE', headers={
+                    'Date': 'Tue, 27 Mar 2007 21:20:27 +0000',
+                    'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
+                    })
+        httpreq = self.download_request(req, self.spider)
+        self.assertEqual(httpreq.headers['Authorization'], \
+                'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
+
+    def test_request_signing6(self):
+        # uploads an object to a CNAME style virtual hosted bucket with metadata.
+        req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \
+                method='PUT', headers={
+                    'User-Agent': 'curl/7.15.5',
+                    'Host': 'static.johnsmith.net:8080',
+                    'Date': 'Tue, 27 Mar 2007 21:06:08 +0000',
+                    'x-amz-acl': 'public-read',
+                    'content-type': 'application/x-download',
+                    'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
+                    'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net',
+                    'X-Amz-Meta-FileChecksum': '0x02661779',
+                    'X-Amz-Meta-ChecksumAlgorithm': 'crc32',
+                    'Content-Disposition': 'attachment; filename=database.dat',
+                    'Content-Encoding': 'gzip',
+                    'Content-Length': '5913339',
+                    })
+        httpreq = self.download_request(req, self.spider)
+        self.assertEqual(httpreq.headers['Authorization'], \
+                'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')