From 9b68c3c1b15eedeb1cfce8c5e7110e20084999fb Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Fri, 3 Sep 2010 16:19:47 -0300 Subject: [PATCH] Add S3 scheme request handler. closes #222 --- scrapy/__init__.py | 7 ++ scrapy/conf/default_settings.py | 1 + scrapy/core/downloader/handlers/s3.py | 34 +++++++++ scrapy/tests/test_downloader_handlers.py | 95 ++++++++++++++++++++++++ 4 files changed, 137 insertions(+) create mode 100644 scrapy/core/downloader/handlers/s3.py diff --git a/scrapy/__init__.py b/scrapy/__init__.py index fd1628ef4..41892ed08 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -26,3 +26,10 @@ except ImportError: pass else: optional_features.add('ssl') + +try: + import boto +except ImportError: + pass +else: + optional_features.add('boto') diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 21f832230..bd6c246f4 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -188,6 +188,7 @@ REQUEST_HANDLERS_BASE = { 'file': 'scrapy.core.downloader.handlers.file.FileRequestHandler', 'http': 'scrapy.core.downloader.handlers.http.HttpRequestHandler', 'https': 'scrapy.core.downloader.handlers.http.HttpRequestHandler', + 's3': 'scrapy.core.downloader.handlers.s3.S3RequestHandler', } REQUESTS_QUEUE_SIZE = 0 diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py new file mode 100644 index 000000000..097f91e54 --- /dev/null +++ b/scrapy/core/downloader/handlers/s3.py @@ -0,0 +1,34 @@ +from scrapy import optional_features +from scrapy.exceptions import NotConfigured +from scrapy.utils.httpobj import urlparse_cached +from scrapy.conf import settings +from .http import HttpRequestHandler + + +class S3RequestHandler(object): + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, \ + httprequesthandler=HttpRequestHandler): + if 'boto' not in optional_features: + raise NotConfigured("missing boto library") + + if not aws_access_key_id: + aws_access_key_id = settings['AWS_ACCESS_KEY_ID'] + if not aws_secret_access_key: + aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY'] + + from boto import connect_s3 + try: + self.conn = connect_s3(aws_access_key_id, aws_secret_access_key) + except Exception, ex: + raise NotConfigured(str(ex)) + self._download_http = httprequesthandler().download_request + + def download_request(self, request, spider): + p = urlparse_cached(request) + scheme = 'https' if request.meta.get('is_secure') else 'http' + url = '%s://%s.s3.amazonaws.com%s' % (scheme, p.hostname, p.path) + httpreq = request.replace(url=url) + self.conn.add_aws_auth_header(httpreq.headers, httpreq.method, \ + '%s/%s' % (p.hostname, p.path)) + return self._download_http(httpreq, spider) diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 6cc79e497..51519cb21 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -12,8 +12,10 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \ from scrapy.core.downloader.webclient import PartialDownloadError from scrapy.core.downloader.handlers.file import FileRequestHandler from scrapy.core.downloader.handlers.http import HttpRequestHandler +from scrapy.core.downloader.handlers.s3 import S3RequestHandler from scrapy.spider import BaseSpider from scrapy.http import Request +from scrapy import optional_features class FileTestCase(unittest.TestCase): @@ -179,3 +181,96 @@ class HttpProxyTestCase(unittest.TestCase): request = Request(self.getURL('path/to/resource')) return self.download_request(request, BaseSpider('foo')).addCallback(_test) + + +class HttpRequestHandlerMock(object): + def download_request(self, request, spider): + return request + +class S3TestCase(unittest.TestCase): + skip = 'boto' not in optional_features and 'missing boto library' + + # test use same example keys than amazon developer guide + # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf + # and the tests described here are the examples from that manual + + AWS_ACCESS_KEY_ID = '0PN5J17HBGZHT7JJ3X82' + AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o' + + def setUp(self): + s3reqh = S3RequestHandler(self.AWS_ACCESS_KEY_ID, \ + self.AWS_SECRET_ACCESS_KEY, \ + httprequesthandler=HttpRequestHandlerMock) + self.download_request = s3reqh.download_request + self.spider = BaseSpider('foo') + + def test_request_signing1(self): + # gets an object from the johnsmith bucket. + req = Request('s3://johnsmith/photos/puppy.jpg', + headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'}) + httpreq = self.download_request(req, self.spider) + self.assertEqual(httpreq.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=') + + def test_request_signing2(self): + # puts an object into the johnsmith bucket. + req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={ + 'Content-Type': 'image/jpeg', + 'Date': 'Tue, 27 Mar 2007 21:15:45 +0000', + 'Content-Length': '94328', + }) + httpreq = self.download_request(req, self.spider) + self.assertEqual(httpreq.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=') + + def test_request_signing3(self): + # lists the content of the johnsmith bucket. + req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \ + method='GET', headers={ + 'User-Agent': 'Mozilla/5.0', + 'Date': 'Tue, 27 Mar 2007 19:42:41 +0000', + }) + httpreq = self.download_request(req, self.spider) + self.assertEqual(httpreq.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=') + + def test_request_signing4(self): + # fetches the access control policy sub-resource for the 'johnsmith' bucket. + req = Request('s3://johnsmith/?acl', \ + method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'}) + httpreq = self.download_request(req, self.spider) + self.assertEqual(httpreq.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') + + def test_request_signing5(self): + # deletes an object from the 'johnsmith' bucket using the + # path-style and Date alternative. + req = Request('s3://johnsmith/photos/puppy.jpg', \ + method='DELETE', headers={ + 'Date': 'Tue, 27 Mar 2007 21:20:27 +0000', + 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000', + }) + httpreq = self.download_request(req, self.spider) + self.assertEqual(httpreq.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') + + def test_request_signing6(self): + # uploads an object to a CNAME style virtual hosted bucket with metadata. + req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \ + method='PUT', headers={ + 'User-Agent': 'curl/7.15.5', + 'Host': 'static.johnsmith.net:8080', + 'Date': 'Tue, 27 Mar 2007 21:06:08 +0000', + 'x-amz-acl': 'public-read', + 'content-type': 'application/x-download', + 'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==', + 'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net', + 'X-Amz-Meta-FileChecksum': '0x02661779', + 'X-Amz-Meta-ChecksumAlgorithm': 'crc32', + 'Content-Disposition': 'attachment; filename=database.dat', + 'Content-Encoding': 'gzip', + 'Content-Length': '5913339', + }) + httpreq = self.download_request(req, self.spider) + self.assertEqual(httpreq.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')