RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Convert request handlers to classes and support NotConfigured. closes #221

This commit is contained in:
Daniel Grana 2010-09-03 16:18:46 -03:00
parent fb69655a9f
commit 30d94b5bf5
5 changed files with 77 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/conf/default_settings.py
View File

@ -185,9 +185,9 @@ REDIRECT_PRIORITY_ADJUST = +2
REQUEST_HANDLERS = {}
REQUEST_HANDLERS_BASE = {
'file': 'scrapy.core.downloader.handlers.file.download_file',
'http': 'scrapy.core.downloader.handlers.http.download_http',
'https': 'scrapy.core.downloader.handlers.http.download_http',
'file': 'scrapy.core.downloader.handlers.file.FileRequestHandler',
'http': 'scrapy.core.downloader.handlers.http.HttpRequestHandler',
'https': 'scrapy.core.downloader.handlers.http.HttpRequestHandler',
}
REQUESTS_QUEUE_SIZE = 0

17
scrapy/core/downloader/handlers/__init__.py
View File

@ -1,6 +1,6 @@
"""Download handlers for different schemes"""
from scrapy.exceptions import NotSupported
from scrapy.exceptions import NotSupported, NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.conf import settings
from scrapy.utils.misc import load_object
@ -10,17 +10,26 @@ class RequestHandlers(object):
def __init__(self):
self._handlers = {}
self._notconfigured = {}
handlers = settings.get('REQUEST_HANDLERS_BASE')
handlers.update(settings.get('REQUEST_HANDLERS', {}))
for scheme, cls in handlers.iteritems():
self._handlers[scheme] = load_object(cls)
for scheme, clspath in handlers.iteritems():
cls = load_object(clspath)
try:
dh = cls()
except NotConfigured, ex:
self._notconfigured[scheme] = str(ex)
else:
self._handlers[scheme] = dh.download_request
def download_request(self, request, spider):
scheme = urlparse_cached(request).scheme
try:
handler = self._handlers[scheme]
except KeyError:
raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url))
msg = self._notconfigured.get(scheme, \
'no handler available for that scheme')
raise NotSupported("Unsupported URL scheme '%s': %s" % (scheme, msg))
return handler(request, spider)

19
scrapy/core/downloader/handlers/file.py
View File

@ -7,14 +7,15 @@ from twisted.internet import defer
from scrapy.core.downloader.responsetypes import responsetypes
def download_file(request, spider):
"""Return a deferred for a file download."""
return defer.maybeDeferred(_all_in_one_read_download_file, request, spider)
class FileRequestHandler(object):
"""file download"""
def _all_in_one_read_download_file(request, spider):
filepath = url2pathname(request.url.split("file://")[1])
with open(filepath) as f:
body = f.read()
respcls = responsetypes.from_args(filename=filepath, body=body)
return respcls(url=request.url, body=body)
def download_request(self, request, spider):
return defer.maybeDeferred(self._one_pass_read, request)
def _one_pass_read(self, request):
filepath = url2pathname(request.url.split("file://")[1])
with open(filepath) as f:
body = f.read()
respcls = responsetypes.from_args(filename=filepath, body=body)
return respcls(url=request.url, body=body)

60
scrapy/core/downloader/handlers/http.py
View File

@ -13,38 +13,42 @@ ssl_supported = 'ssl' in optional_features
if ssl_supported:
from twisted.internet.ssl import ClientContextFactory
HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
def _create_factory(request, spider):
def _download_signals(response):
send_catch_log(signal=signals.request_uploaded, request=request, \
spider=spider)
send_catch_log(signal=signals.response_downloaded, response=response, \
spider=spider)
return response
timeout = getattr(spider, "download_timeout", None) or default_timeout
factory = HTTPClientFactory(request, timeout)
factory.deferred.addCallbacks(_download_signals)
return factory
DOWNLOAD_TIMEOUT = settings.getint('DOWNLOAD_TIMEOUT')
def _connect(factory):
host, port = factory.host, factory.port
if factory.scheme == 'https':
if ssl_supported:
return reactor.connectSSL(host, port, factory, ClientContextFactory())
raise NotSupported("HTTPS not supported: install pyopenssl library")
else:
return reactor.connectTCP(host, port, factory)
class HttpRequestHandler(object):
def __init__(self, httpclientfactory=HTTPClientFactory, \
download_timeout=DOWNLOAD_TIMEOUT):
self.httpclientfactory = httpclientfactory
self.download_timeout = download_timeout
def download_http(request, spider):
"""Return a deferred for the HTTP download"""
factory = _create_factory(request, spider)
_connect(factory)
return factory.deferred
def download_request(self, request, spider):
"""Return a deferred for the HTTP download"""
factory = self._create_factory(request, spider)
self._connect(factory)
return factory.deferred
def _create_factory(self, request, spider):
def _download_signals(response):
send_catch_log(signal=signals.request_uploaded, request=request, \
spider=spider)
send_catch_log(signal=signals.response_downloaded, response=response, \
spider=spider)
return response
timeout = getattr(spider, "download_timeout", None) or self.download_timeout
factory = self.httpclientfactory(request, timeout)
factory.deferred.addCallbacks(_download_signals)
return factory
def _connect(self, factory):
host, port = factory.host, factory.port
if factory.scheme == 'https':
if ssl_supported:
return reactor.connectSSL(host, port, factory, \
ClientContextFactory())
raise NotSupported("HTTPS not supported: install pyopenssl library")
else:
return reactor.connectTCP(host, port, factory)

35
scrapy/tests/test_downloader_handlers.py
View File

@ -10,8 +10,8 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
PayloadResource, BrokenDownloadResource
from scrapy.core.downloader.webclient import PartialDownloadError
from scrapy.core.downloader.handlers.file import download_file
from scrapy.core.downloader.handlers.http import download_http
from scrapy.core.downloader.handlers.file import FileRequestHandler
from scrapy.core.downloader.handlers.http import HttpRequestHandler
from scrapy.spider import BaseSpider
from scrapy.http import Request
@ -23,6 +23,7 @@ class FileTestCase(unittest.TestCase):
fd = open(self.tmpname + '^', 'w')
fd.write('0123456789')
fd.close()
self.download_request = FileRequestHandler().download_request
def test_download(self):
def _test(response):
@ -32,11 +33,11 @@ class FileTestCase(unittest.TestCase):
request = Request('file://%s' % self.tmpname + '^')
assert request.url.upper().endswith('%5E')
return download_file(request, BaseSpider('foo')).addCallback(_test)
return self.download_request(request, BaseSpider('foo')).addCallback(_test)
def test_non_existent(self):
request = Request('file://%s' % self.mktemp())
d = download_file(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
return self.assertFailure(d, IOError)
@ -57,6 +58,7 @@ class HttpTestCase(unittest.TestCase):
self.wrapper = WrappingFactory(self.site)
self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1')
self.portno = self.port.getHost().port
self.download_request = HttpRequestHandler().download_request
def tearDown(self):
return self.port.stopListening()
@ -66,28 +68,28 @@ class HttpTestCase(unittest.TestCase):
def test_download(self):
request = Request(self.getURL('file'))
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
d.addCallback(lambda r: r.body)
d.addCallback(self.assertEquals, "0123456789")
return d
def test_download_head(self):
request = Request(self.getURL('file'), method='HEAD')
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
d.addCallback(lambda r: r.body)
d.addCallback(self.assertEquals, '')
return d
def test_redirect_status(self):
request = Request(self.getURL('redirect'))
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
d.addCallback(lambda r: r.status)
d.addCallback(self.assertEquals, 302)
return d
def test_redirect_status_head(self):
request = Request(self.getURL('redirect'), method='HEAD')
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
d.addCallback(lambda r: r.status)
d.addCallback(self.assertEquals, 302)
return d
@ -96,7 +98,7 @@ class HttpTestCase(unittest.TestCase):
spider = BaseSpider('foo')
spider.download_timeout = 0.000001
request = Request(self.getURL('wait'))
d = download_http(request, spider)
d = self.download_request(request, spider)
return self.assertFailure(d, defer.TimeoutError)
def test_host_header_not_in_request_headers(self):
@ -105,7 +107,7 @@ class HttpTestCase(unittest.TestCase):
self.assertEquals(request.headers, {})
request = Request(self.getURL('host'))
return download_http(request, BaseSpider('foo')).addCallback(_test)
return self.download_request(request, BaseSpider('foo')).addCallback(_test)
def test_host_header_seted_in_request_headers(self):
def _test(response):
@ -113,9 +115,9 @@ class HttpTestCase(unittest.TestCase):
self.assertEquals(request.headers.get('Host'), 'example.com')
request = Request(self.getURL('host'), headers={'Host': 'example.com'})
return download_http(request, BaseSpider('foo')).addCallback(_test)
return self.download_request(request, BaseSpider('foo')).addCallback(_test)
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
d.addCallback(lambda r: r.body)
d.addCallback(self.assertEquals, 'example.com')
return d
@ -123,14 +125,14 @@ class HttpTestCase(unittest.TestCase):
def test_payload(self):
body = '1'*100 # PayloadResource requires body length to be 100
request = Request(self.getURL('payload'), method='POST', body=body)
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
d.addCallback(lambda r: r.body)
d.addCallback(self.assertEquals, body)
return d
def test_broken_download(self):
request = Request(self.getURL('broken'))
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
return self.assertFailure(d, PartialDownloadError)
@ -151,6 +153,7 @@ class HttpProxyTestCase(unittest.TestCase):
wrapper = WrappingFactory(site)
self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
self.portno = self.port.getHost().port
self.download_request = HttpRequestHandler().download_request
def tearDown(self):
return self.port.stopListening()
@ -166,7 +169,7 @@ class HttpProxyTestCase(unittest.TestCase):
http_proxy = self.getURL('')
request = Request('https://example.com', meta={'proxy': http_proxy})
return download_http(request, BaseSpider('foo')).addCallback(_test)
return self.download_request(request, BaseSpider('foo')).addCallback(_test)
def test_download_without_proxy(self):
def _test(response):
@ -175,4 +178,4 @@ class HttpProxyTestCase(unittest.TestCase):
self.assertEquals(response.body, '/path/to/resource')
request = Request(self.getURL('path/to/resource'))
return download_http(request, BaseSpider('foo')).addCallback(_test)
return self.download_request(request, BaseSpider('foo')).addCallback(_test)