diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index eed71eda9..21f832230 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -185,9 +185,9 @@ REDIRECT_PRIORITY_ADJUST = +2 REQUEST_HANDLERS = {} REQUEST_HANDLERS_BASE = { - 'file': 'scrapy.core.downloader.handlers.file.download_file', - 'http': 'scrapy.core.downloader.handlers.http.download_http', - 'https': 'scrapy.core.downloader.handlers.http.download_http', + 'file': 'scrapy.core.downloader.handlers.file.FileRequestHandler', + 'http': 'scrapy.core.downloader.handlers.http.HttpRequestHandler', + 'https': 'scrapy.core.downloader.handlers.http.HttpRequestHandler', } REQUESTS_QUEUE_SIZE = 0 diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index 6a1556768..c89b943a2 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,6 +1,6 @@ """Download handlers for different schemes""" -from scrapy.exceptions import NotSupported +from scrapy.exceptions import NotSupported, NotConfigured from scrapy.utils.httpobj import urlparse_cached from scrapy.conf import settings from scrapy.utils.misc import load_object @@ -10,17 +10,26 @@ class RequestHandlers(object): def __init__(self): self._handlers = {} + self._notconfigured = {} handlers = settings.get('REQUEST_HANDLERS_BASE') handlers.update(settings.get('REQUEST_HANDLERS', {})) - for scheme, cls in handlers.iteritems(): - self._handlers[scheme] = load_object(cls) + for scheme, clspath in handlers.iteritems(): + cls = load_object(clspath) + try: + dh = cls() + except NotConfigured, ex: + self._notconfigured[scheme] = str(ex) + else: + self._handlers[scheme] = dh.download_request def download_request(self, request, spider): scheme = urlparse_cached(request).scheme try: handler = self._handlers[scheme] except KeyError: - raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url)) + msg = self._notconfigured.get(scheme, \ + 'no handler available for that scheme') + raise NotSupported("Unsupported URL scheme '%s': %s" % (scheme, msg)) return handler(request, spider) diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py index 98bfa3342..daa1ed034 100644 --- a/scrapy/core/downloader/handlers/file.py +++ b/scrapy/core/downloader/handlers/file.py @@ -7,14 +7,15 @@ from twisted.internet import defer from scrapy.core.downloader.responsetypes import responsetypes -def download_file(request, spider): - """Return a deferred for a file download.""" - return defer.maybeDeferred(_all_in_one_read_download_file, request, spider) +class FileRequestHandler(object): + """file download""" -def _all_in_one_read_download_file(request, spider): - filepath = url2pathname(request.url.split("file://")[1]) - with open(filepath) as f: - body = f.read() - respcls = responsetypes.from_args(filename=filepath, body=body) - return respcls(url=request.url, body=body) + def download_request(self, request, spider): + return defer.maybeDeferred(self._one_pass_read, request) + def _one_pass_read(self, request): + filepath = url2pathname(request.url.split("file://")[1]) + with open(filepath) as f: + body = f.read() + respcls = responsetypes.from_args(filename=filepath, body=body) + return respcls(url=request.url, body=body) diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 7e726a0f3..3100858d0 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -13,38 +13,42 @@ ssl_supported = 'ssl' in optional_features if ssl_supported: from twisted.internet.ssl import ClientContextFactory - HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) -default_timeout = settings.getint('DOWNLOAD_TIMEOUT') - -def _create_factory(request, spider): - def _download_signals(response): - send_catch_log(signal=signals.request_uploaded, request=request, \ - spider=spider) - send_catch_log(signal=signals.response_downloaded, response=response, \ - spider=spider) - return response - - timeout = getattr(spider, "download_timeout", None) or default_timeout - factory = HTTPClientFactory(request, timeout) - factory.deferred.addCallbacks(_download_signals) - return factory +DOWNLOAD_TIMEOUT = settings.getint('DOWNLOAD_TIMEOUT') -def _connect(factory): - host, port = factory.host, factory.port - if factory.scheme == 'https': - if ssl_supported: - return reactor.connectSSL(host, port, factory, ClientContextFactory()) - raise NotSupported("HTTPS not supported: install pyopenssl library") - else: - return reactor.connectTCP(host, port, factory) +class HttpRequestHandler(object): + def __init__(self, httpclientfactory=HTTPClientFactory, \ + download_timeout=DOWNLOAD_TIMEOUT): + self.httpclientfactory = httpclientfactory + self.download_timeout = download_timeout -def download_http(request, spider): - """Return a deferred for the HTTP download""" - factory = _create_factory(request, spider) - _connect(factory) - return factory.deferred + def download_request(self, request, spider): + """Return a deferred for the HTTP download""" + factory = self._create_factory(request, spider) + self._connect(factory) + return factory.deferred + def _create_factory(self, request, spider): + def _download_signals(response): + send_catch_log(signal=signals.request_uploaded, request=request, \ + spider=spider) + send_catch_log(signal=signals.response_downloaded, response=response, \ + spider=spider) + return response + timeout = getattr(spider, "download_timeout", None) or self.download_timeout + factory = self.httpclientfactory(request, timeout) + factory.deferred.addCallbacks(_download_signals) + return factory + + def _connect(self, factory): + host, port = factory.host, factory.port + if factory.scheme == 'https': + if ssl_supported: + return reactor.connectSSL(host, port, factory, \ + ClientContextFactory()) + raise NotSupported("HTTPS not supported: install pyopenssl library") + else: + return reactor.connectTCP(host, port, factory) diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 4c08cf021..6cc79e497 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -10,8 +10,8 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \ PayloadResource, BrokenDownloadResource from scrapy.core.downloader.webclient import PartialDownloadError -from scrapy.core.downloader.handlers.file import download_file -from scrapy.core.downloader.handlers.http import download_http +from scrapy.core.downloader.handlers.file import FileRequestHandler +from scrapy.core.downloader.handlers.http import HttpRequestHandler from scrapy.spider import BaseSpider from scrapy.http import Request @@ -23,6 +23,7 @@ class FileTestCase(unittest.TestCase): fd = open(self.tmpname + '^', 'w') fd.write('0123456789') fd.close() + self.download_request = FileRequestHandler().download_request def test_download(self): def _test(response): @@ -32,11 +33,11 @@ class FileTestCase(unittest.TestCase): request = Request('file://%s' % self.tmpname + '^') assert request.url.upper().endswith('%5E') - return download_file(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, BaseSpider('foo')).addCallback(_test) def test_non_existent(self): request = Request('file://%s' % self.mktemp()) - d = download_file(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) return self.assertFailure(d, IOError) @@ -57,6 +58,7 @@ class HttpTestCase(unittest.TestCase): self.wrapper = WrappingFactory(self.site) self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1') self.portno = self.port.getHost().port + self.download_request = HttpRequestHandler().download_request def tearDown(self): return self.port.stopListening() @@ -66,28 +68,28 @@ class HttpTestCase(unittest.TestCase): def test_download(self): request = Request(self.getURL('file')) - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, "0123456789") return d def test_download_head(self): request = Request(self.getURL('file'), method='HEAD') - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, '') return d def test_redirect_status(self): request = Request(self.getURL('redirect')) - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) d.addCallback(lambda r: r.status) d.addCallback(self.assertEquals, 302) return d def test_redirect_status_head(self): request = Request(self.getURL('redirect'), method='HEAD') - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) d.addCallback(lambda r: r.status) d.addCallback(self.assertEquals, 302) return d @@ -96,7 +98,7 @@ class HttpTestCase(unittest.TestCase): spider = BaseSpider('foo') spider.download_timeout = 0.000001 request = Request(self.getURL('wait')) - d = download_http(request, spider) + d = self.download_request(request, spider) return self.assertFailure(d, defer.TimeoutError) def test_host_header_not_in_request_headers(self): @@ -105,7 +107,7 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers, {}) request = Request(self.getURL('host')) - return download_http(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, BaseSpider('foo')).addCallback(_test) def test_host_header_seted_in_request_headers(self): def _test(response): @@ -113,9 +115,9 @@ class HttpTestCase(unittest.TestCase): self.assertEquals(request.headers.get('Host'), 'example.com') request = Request(self.getURL('host'), headers={'Host': 'example.com'}) - return download_http(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, BaseSpider('foo')).addCallback(_test) - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, 'example.com') return d @@ -123,14 +125,14 @@ class HttpTestCase(unittest.TestCase): def test_payload(self): body = '1'*100 # PayloadResource requires body length to be 100 request = Request(self.getURL('payload'), method='POST', body=body) - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) d.addCallback(lambda r: r.body) d.addCallback(self.assertEquals, body) return d def test_broken_download(self): request = Request(self.getURL('broken')) - d = download_http(request, BaseSpider('foo')) + d = self.download_request(request, BaseSpider('foo')) return self.assertFailure(d, PartialDownloadError) @@ -151,6 +153,7 @@ class HttpProxyTestCase(unittest.TestCase): wrapper = WrappingFactory(site) self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1') self.portno = self.port.getHost().port + self.download_request = HttpRequestHandler().download_request def tearDown(self): return self.port.stopListening() @@ -166,7 +169,7 @@ class HttpProxyTestCase(unittest.TestCase): http_proxy = self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) - return download_http(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, BaseSpider('foo')).addCallback(_test) def test_download_without_proxy(self): def _test(response): @@ -175,4 +178,4 @@ class HttpProxyTestCase(unittest.TestCase): self.assertEquals(response.body, '/path/to/resource') request = Request(self.getURL('path/to/resource')) - return download_http(request, BaseSpider('foo')).addCallback(_test) + return self.download_request(request, BaseSpider('foo')).addCallback(_test)