diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 030b6c7f9..cc9152892 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -31,3 +31,10 @@ except ImportError: pass else: optional_features.add('ssl') + +try: + import boto +except ImportError: + pass +else: + optional_features.add('boto') diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 8a2b180ef..b6fac9107 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -185,9 +185,10 @@ REDIRECT_PRIORITY_ADJUST = +2 REQUEST_HANDLERS = {} REQUEST_HANDLERS_BASE = { - 'file': 'scrapy.core.downloader.handlers.file.download_file', - 'http': 'scrapy.core.downloader.handlers.http.download_http', - 'https': 'scrapy.core.downloader.handlers.http.download_http', + 'file': 'scrapy.core.downloader.handlers.file.FileRequestHandler', + 'http': 'scrapy.core.downloader.handlers.http.HttpRequestHandler', + 'https': 'scrapy.core.downloader.handlers.http.HttpRequestHandler', + 's3': 'scrapy.core.downloader.handlers.s3.S3RequestHandler', } REQUESTS_QUEUE_SIZE = 0 diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 6a1556768..c89b943a2 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,6 +1,6 @@ """Download handlers for different schemes""" -from scrapy.exceptions import NotSupported +from scrapy.exceptions import NotSupported, NotConfigured from scrapy.utils.httpobj import urlparse_cached from scrapy.conf import settings from scrapy.utils.misc import load_object @@ -10,17 +10,26 @@ class RequestHandlers(object): def __init__(self): self._handlers = {} + self._notconfigured = {} handlers = settings.get('REQUEST_HANDLERS_BASE') handlers.update(settings.get('REQUEST_HANDLERS', {})) - for scheme, cls in handlers.iteritems(): - self._handlers[scheme] = load_object(cls) + for scheme, clspath in handlers.iteritems(): + cls = load_object(clspath) + try: + dh = cls() + except NotConfigured, ex: + self._notconfigured[scheme] = str(ex) + else: + self._handlers[scheme] = dh.download_request def download_request(self, request, spider): scheme = urlparse_cached(request).scheme try: handler = self._handlers[scheme] except KeyError: - raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url)) + msg = self._notconfigured.get(scheme, \ + 'no handler available for that scheme') + raise NotSupported("Unsupported URL scheme '%s': %s" % (scheme, msg)) return handler(request, spider) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 98bfa3342..daa1ed034 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -7,14 +7,15 @@ from twisted.internet import defer from scrapy.core.downloader.responsetypes import responsetypes -def download_file(request, spider): - """Return a deferred for a file download.""" - return defer.maybeDeferred(_all_in_one_read_download_file, request, spider) +class FileRequestHandler(object): + """file download""" -def _all_in_one_read_download_file(request, spider): - filepath = url2pathname(request.url.split("file://")[1]) - with open(filepath) as f: - body = f.read() - respcls = responsetypes.from_args(filename=filepath, body=body) - return respcls(url=request.url, body=body) + def download_request(self, request, spider): + return defer.maybeDeferred(self._one_pass_read, request) + def _one_pass_read(self, request): + filepath = url2pathname(request.url.split("file://")[1]) + with open(filepath) as f: + body = f.read() + respcls = responsetypes.from_args(filename=filepath, body=body) + return respcls(url=request.url, body=body) diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 7e726a0f3..3100858d0 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -13,38 +13,42 @@ ssl_supported = 'ssl' in optional_features if ssl_supported: from twisted.internet.ssl import ClientContextFactory - HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) -default_timeout = settings.getint('DOWNLOAD_TIMEOUT') - -def _create_factory(request, spider): - def _download_signals(response): - send_catch_log(signal=signals.request_uploaded, request=request, \ - spider=spider) - send_catch_log(signal=signals.response_downloaded, response=response, \ - spider=spider) - return response - - timeout = getattr(spider, "download_timeout", None) or default_timeout - factory = HTTPClientFactory(request, timeout) - factory.deferred.addCallbacks(_download_signals) - return factory +DOWNLOAD_TIMEOUT = settings.getint('DOWNLOAD_TIMEOUT') -def _connect(factory): - host, port = factory.host, factory.port - if factory.scheme == 'https': - if ssl_supported: - return reactor.connectSSL(host, port, factory, ClientContextFactory()) - raise NotSupported("HTTPS not supported: install pyopenssl library") - else: - return reactor.connectTCP(host, port, factory) +class HttpRequestHandler(object): + def __init__(self, httpclientfactory=HTTPClientFactory, \ + download_timeout=DOWNLOAD_TIMEOUT): + self.httpclientfactory = httpclientfactory + self.download_timeout = download_timeout -def download_http(request, spider): - """Return a deferred for the HTTP download""" - factory = _create_factory(request, spider) - _connect(factory) - return factory.deferred + def download_request(self, request, spider): + """Return a deferred for the HTTP download""" + factory = self._create_factory(request, spider) + self._connect(factory) + return factory.deferred + def _create_factory(self, request, spider): + def _download_signals(response): + send_catch_log(signal=signals.request_uploaded, request=request, \ + spider=spider) + send_catch_log(signal=signals.response_downloaded, response=response, \ + spider=spider) + return response + timeout = getattr(spider, "download_timeout", None) or self.download_timeout + factory = self.httpclientfactory(request, timeout) + factory.deferred.addCallbacks(_download_signals) + return factory + + def _connect(self, factory): + host, port = factory.host, factory.port + if factory.scheme == 'https': + if ssl_supported: + return reactor.connectSSL(host, port, factory, \ + ClientContextFactory()) + raise NotSupported("HTTPS not supported: install pyopenssl library") + else: + return reactor.connectTCP(host, port, factory) diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py new file mode 100644 index 000000000..097f91e54 --- /dev/null +++ b/scrapy/core/downloader/handlers/s3.py @@ -0,0 +1,34 @@ +from scrapy import optional_features +from scrapy.exceptions import NotConfigured +from scrapy.utils.httpobj import urlparse_cached +from scrapy.conf import settings +from .http import HttpRequestHandler + + +class S3RequestHandler(object): + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, \ + httprequesthandler=HttpRequestHandler): + if 'boto' not in optional_features: + raise NotConfigured("missing boto library") + + if not aws_access_key_id: + aws_access_key_id = settings['AWS_ACCESS_KEY_ID'] + if not aws_secret_access_key: + aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY'] + + from boto import connect_s3 + try: + self.conn = connect_s3(aws_access_key_id, aws_secret_access_key) + except Exception, ex: + raise NotConfigured(str(ex)) + self._download_http = httprequesthandler().download_request + + def download_request(self, request, spider): + p = urlparse_cached(request) + scheme = 'https' if request.meta.get('is_secure') else 'http' + url = '%s://%s.s3.amazonaws.com%s' % (scheme, p.hostname, p.path) + httpreq = request.replace(url=url) + self.conn.add_aws_auth_header(httpreq.headers, httpreq.method, \ + '%s/%s' % (p.hostname, p.path)) + return self._download_http(httpreq, spider) diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 4c08cf021..51519cb21 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -10,10 +10,12 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \ PayloadResource, BrokenDownloadResource from scrapy.core.downloader.webclient import PartialDownloadError -from scrapy.core.downloader.handlers.file import download_file -from scrapy.core.downloader.handlers.http import download_http +from scrapy.core.downloader.handlers.file import FileRequestHandler +from scrapy.core.downloader.handlers.http import HttpRequestHandler +from scrapy.core.downloader.handlers.s3 import S3RequestHandler from scrapy.spider import BaseSpider from scrapy.http import Request +from scrapy import optional_features class FileTestCase(unittest.TestCase): @@ -23,6 +25,7 @@ class FileTestCase(unittest.TestCase): fd = open(self.tmpname + '^', 'w') fd.write('0123456789') fd.close() + self.download_request = FileRequestHandler().download_request def test_download(self): def _test(response): @@ -32,11 +35,11 @@ class FileTestCase(unittest.TestCase): request = Request('file://%s' % self.tmpname + '^') assert request.url.upper().endswith('%5E') - return download_file(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, BaseSpider('foo')).addCallback(_test) def test_non_existent(self): request = Request('file://%s' % self.mktemp()) - d = download_file(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) return self.assertFailure(d, IOError) @@ -57,6 +60,7 @@ class HttpTestCase(unittest.TestCase): self.wrapper = WrappingFactory(self.site) self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1') self.portno = self.port.getHost().port + self.download_request = HttpRequestHandler().download_request def tearDown(self): return self.port.stopListening() @@ -66,28 +70,28 @@ class HttpTestCase(unittest.TestCase): def test_download(self): request = Request(self.getURL('file')) - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, "0123456789") return d def test_download_head(self): request = Request(self.getURL('file'), method='HEAD') - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, '') return d def test_redirect_status(self): request = Request(self.getURL('redirect')) - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) d.addCallback(lambda r: r.status) d.addCallback(self.assertEquals, 302) return d def test_redirect_status_head(self): request = Request(self.getURL('redirect'), method='HEAD') - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) d.addCallback(lambda r: r.status) d.addCallback(self.assertEquals, 302) return d @@ -96,7 +100,7 @@ class HttpTestCase(unittest.TestCase): spider = BaseSpider('foo') spider.download_timeout = 0.000001 request = Request(self.getURL('wait')) - d = download_http(request, spider) + d = self.download_request(request, spider) return self.assertFailure(d, defer.TimeoutError) def test_host_header_not_in_request_headers(self): @@ -105,7 +109,7 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers, {}) request = Request(self.getURL('host')) - return download_http(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, BaseSpider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): def _test(response): @@ -113,9 +117,9 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers.get('Host'), 'example.com') request = Request(self.getURL('host'), headers={'Host': 'example.com'}) - return download_http(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, BaseSpider('foo')).addCallback(_test) - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, 'example.com') return d @@ -123,14 +127,14 @@ class HttpTestCase(unittest.TestCase): def test_payload(self): body = '1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, body) return d def test_broken_download(self): request = Request(self.getURL('broken')) - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) return self.assertFailure(d, PartialDownloadError) @@ -151,6 +155,7 @@ class HttpProxyTestCase(unittest.TestCase): wrapper = WrappingFactory(site) self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1') self.portno = self.port.getHost().port + self.download_request = HttpRequestHandler().download_request def tearDown(self): return self.port.stopListening() @@ -166,7 +171,7 @@ class HttpProxyTestCase(unittest.TestCase): http_proxy = self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) - return download_http(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, BaseSpider('foo')).addCallback(_test) def test_download_without_proxy(self): def _test(response): @@ -175,4 +180,97 @@ class HttpProxyTestCase(unittest.TestCase): self.assertEquals(response.body, '/path/to/resource') request = Request(self.getURL('path/to/resource')) - return download_http(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, BaseSpider('foo')).addCallback(_test) + + +class HttpRequestHandlerMock(object): + def download_request(self, request, spider): + return request + +class S3TestCase(unittest.TestCase): + skip = 'boto' not in optional_features and 'missing boto library' + + # test use same example keys than amazon developer guide + # http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf + # and the tests described here are the examples from that manual + + AWS_ACCESS_KEY_ID = '0PN5J17HBGZHT7JJ3X82' + AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o' + + def setUp(self): + s3reqh = S3RequestHandler(self.AWS_ACCESS_KEY_ID, \ + self.AWS_SECRET_ACCESS_KEY, \ + httprequesthandler=HttpRequestHandlerMock) + self.download_request = s3reqh.download_request + self.spider = BaseSpider('foo') + + def test_request_signing1(self): + # gets an object from the johnsmith bucket. + req = Request('s3://johnsmith/photos/puppy.jpg', + headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'}) + httpreq = self.download_request(req, self.spider) + self.assertEqual(httpreq.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=') + + def test_request_signing2(self): + # puts an object into the johnsmith bucket. + req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={ + 'Content-Type': 'image/jpeg', + 'Date': 'Tue, 27 Mar 2007 21:15:45 +0000', + 'Content-Length': '94328', + }) + httpreq = self.download_request(req, self.spider) + self.assertEqual(httpreq.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=') + + def test_request_signing3(self): + # lists the content of the johnsmith bucket. + req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \ + method='GET', headers={ + 'User-Agent': 'Mozilla/5.0', + 'Date': 'Tue, 27 Mar 2007 19:42:41 +0000', + }) + httpreq = self.download_request(req, self.spider) + self.assertEqual(httpreq.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=') + + def test_request_signing4(self): + # fetches the access control policy sub-resource for the 'johnsmith' bucket. + req = Request('s3://johnsmith/?acl', \ + method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'}) + httpreq = self.download_request(req, self.spider) + self.assertEqual(httpreq.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=') + + def test_request_signing5(self): + # deletes an object from the 'johnsmith' bucket using the + # path-style and Date alternative. + req = Request('s3://johnsmith/photos/puppy.jpg', \ + method='DELETE', headers={ + 'Date': 'Tue, 27 Mar 2007 21:20:27 +0000', + 'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000', + }) + httpreq = self.download_request(req, self.spider) + self.assertEqual(httpreq.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=') + + def test_request_signing6(self): + # uploads an object to a CNAME style virtual hosted bucket with metadata. + req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \ + method='PUT', headers={ + 'User-Agent': 'curl/7.15.5', + 'Host': 'static.johnsmith.net:8080', + 'Date': 'Tue, 27 Mar 2007 21:06:08 +0000', + 'x-amz-acl': 'public-read', + 'content-type': 'application/x-download', + 'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==', + 'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net', + 'X-Amz-Meta-FileChecksum': '0x02661779', + 'X-Amz-Meta-ChecksumAlgorithm': 'crc32', + 'Content-Disposition': 'attachment; filename=database.dat', + 'Content-Encoding': 'gzip', + 'Content-Length': '5913339', + }) + httpreq = self.download_request(req, self.spider) + self.assertEqual(httpreq.headers['Authorization'], \ + 'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')