RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Automatic merge

This commit is contained in:
Pablo Hoffman 2010-09-03 17:46:36 -03:00
parent 37e9c5d78e 9b68c3c1b1
commit 9cfa8edd14
7 changed files with 214 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapy/__init__.py
View File

@ -31,3 +31,10 @@ except ImportError:
pass
else:
optional_features.add('ssl')
try:
import boto
except ImportError:
pass
else:
optional_features.add('boto')

7
scrapy/conf/default_settings.py
View File

@ -185,9 +185,10 @@ REDIRECT_PRIORITY_ADJUST = +2
REQUEST_HANDLERS = {}
REQUEST_HANDLERS_BASE = {
'file': 'scrapy.core.downloader.handlers.file.download_file',
'http': 'scrapy.core.downloader.handlers.http.download_http',
'https': 'scrapy.core.downloader.handlers.http.download_http',
'file': 'scrapy.core.downloader.handlers.file.FileRequestHandler',
'http': 'scrapy.core.downloader.handlers.http.HttpRequestHandler',
'https': 'scrapy.core.downloader.handlers.http.HttpRequestHandler',
's3': 'scrapy.core.downloader.handlers.s3.S3RequestHandler',
}
REQUESTS_QUEUE_SIZE = 0

17
scrapy/core/downloader/handlers/__init__.py
View File

@ -1,6 +1,6 @@
"""Download handlers for different schemes"""
from scrapy.exceptions import NotSupported
from scrapy.exceptions import NotSupported, NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.conf import settings
from scrapy.utils.misc import load_object
@ -10,17 +10,26 @@ class RequestHandlers(object):
def __init__(self):
self._handlers = {}
self._notconfigured = {}
handlers = settings.get('REQUEST_HANDLERS_BASE')
handlers.update(settings.get('REQUEST_HANDLERS', {}))
for scheme, cls in handlers.iteritems():
self._handlers[scheme] = load_object(cls)
for scheme, clspath in handlers.iteritems():
cls = load_object(clspath)
try:
dh = cls()
except NotConfigured, ex:
self._notconfigured[scheme] = str(ex)
else:
self._handlers[scheme] = dh.download_request
def download_request(self, request, spider):
scheme = urlparse_cached(request).scheme
try:
handler = self._handlers[scheme]
except KeyError:
raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url))
msg = self._notconfigured.get(scheme, \
'no handler available for that scheme')
raise NotSupported("Unsupported URL scheme '%s': %s" % (scheme, msg))
return handler(request, spider)

19
scrapy/core/downloader/handlers/file.py
View File

@ -7,14 +7,15 @@ from twisted.internet import defer
from scrapy.core.downloader.responsetypes import responsetypes
def download_file(request, spider):
"""Return a deferred for a file download."""
return defer.maybeDeferred(_all_in_one_read_download_file, request, spider)
class FileRequestHandler(object):
"""file download"""
def _all_in_one_read_download_file(request, spider):
filepath = url2pathname(request.url.split("file://")[1])
with open(filepath) as f:
body = f.read()
respcls = responsetypes.from_args(filename=filepath, body=body)
return respcls(url=request.url, body=body)
def download_request(self, request, spider):
return defer.maybeDeferred(self._one_pass_read, request)
def _one_pass_read(self, request):
filepath = url2pathname(request.url.split("file://")[1])
with open(filepath) as f:
body = f.read()
respcls = responsetypes.from_args(filename=filepath, body=body)
return respcls(url=request.url, body=body)

60
scrapy/core/downloader/handlers/http.py
View File

@ -13,38 +13,42 @@ ssl_supported = 'ssl' in optional_features
if ssl_supported:
from twisted.internet.ssl import ClientContextFactory
HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
def _create_factory(request, spider):
def _download_signals(response):
send_catch_log(signal=signals.request_uploaded, request=request, \
spider=spider)
send_catch_log(signal=signals.response_downloaded, response=response, \
spider=spider)
return response
timeout = getattr(spider, "download_timeout", None) or default_timeout
factory = HTTPClientFactory(request, timeout)
factory.deferred.addCallbacks(_download_signals)
return factory
DOWNLOAD_TIMEOUT = settings.getint('DOWNLOAD_TIMEOUT')
def _connect(factory):
host, port = factory.host, factory.port
if factory.scheme == 'https':
if ssl_supported:
return reactor.connectSSL(host, port, factory, ClientContextFactory())
raise NotSupported("HTTPS not supported: install pyopenssl library")
else:
return reactor.connectTCP(host, port, factory)
class HttpRequestHandler(object):
def __init__(self, httpclientfactory=HTTPClientFactory, \
download_timeout=DOWNLOAD_TIMEOUT):
self.httpclientfactory = httpclientfactory
self.download_timeout = download_timeout
def download_http(request, spider):
"""Return a deferred for the HTTP download"""
factory = _create_factory(request, spider)
_connect(factory)
return factory.deferred
def download_request(self, request, spider):
"""Return a deferred for the HTTP download"""
factory = self._create_factory(request, spider)
self._connect(factory)
return factory.deferred
def _create_factory(self, request, spider):
def _download_signals(response):
send_catch_log(signal=signals.request_uploaded, request=request, \
spider=spider)
send_catch_log(signal=signals.response_downloaded, response=response, \
spider=spider)
return response
timeout = getattr(spider, "download_timeout", None) or self.download_timeout
factory = self.httpclientfactory(request, timeout)
factory.deferred.addCallbacks(_download_signals)
return factory
def _connect(self, factory):
host, port = factory.host, factory.port
if factory.scheme == 'https':
if ssl_supported:
return reactor.connectSSL(host, port, factory, \
ClientContextFactory())
raise NotSupported("HTTPS not supported: install pyopenssl library")
else:
return reactor.connectTCP(host, port, factory)

34
scrapy/core/downloader/handlers/s3.py Normal file
View File

@ -0,0 +1,34 @@
from scrapy import optional_features
from scrapy.exceptions import NotConfigured
from scrapy.utils.httpobj import urlparse_cached
from scrapy.conf import settings
from .http import HttpRequestHandler
class S3RequestHandler(object):
def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, \
httprequesthandler=HttpRequestHandler):
if 'boto' not in optional_features:
raise NotConfigured("missing boto library")
if not aws_access_key_id:
aws_access_key_id = settings['AWS_ACCESS_KEY_ID']
if not aws_secret_access_key:
aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']
from boto import connect_s3
try:
self.conn = connect_s3(aws_access_key_id, aws_secret_access_key)
except Exception, ex:
raise NotConfigured(str(ex))
self._download_http = httprequesthandler().download_request
def download_request(self, request, spider):
p = urlparse_cached(request)
scheme = 'https' if request.meta.get('is_secure') else 'http'
url = '%s://%s.s3.amazonaws.com%s' % (scheme, p.hostname, p.path)
httpreq = request.replace(url=url)
self.conn.add_aws_auth_header(httpreq.headers, httpreq.method, \
'%s/%s' % (p.hostname, p.path))
return self._download_http(httpreq, spider)

130
scrapy/tests/test_downloader_handlers.py
View File

@ -10,10 +10,12 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
PayloadResource, BrokenDownloadResource
from scrapy.core.downloader.webclient import PartialDownloadError
from scrapy.core.downloader.handlers.file import download_file
from scrapy.core.downloader.handlers.http import download_http
from scrapy.core.downloader.handlers.file import FileRequestHandler
from scrapy.core.downloader.handlers.http import HttpRequestHandler
from scrapy.core.downloader.handlers.s3 import S3RequestHandler
from scrapy.spider import BaseSpider
from scrapy.http import Request
from scrapy import optional_features
class FileTestCase(unittest.TestCase):
@ -23,6 +25,7 @@ class FileTestCase(unittest.TestCase):
fd = open(self.tmpname + '^', 'w')
fd.write('0123456789')
fd.close()
self.download_request = FileRequestHandler().download_request
def test_download(self):
def _test(response):
@ -32,11 +35,11 @@ class FileTestCase(unittest.TestCase):
request = Request('file://%s' % self.tmpname + '^')
assert request.url.upper().endswith('%5E')
return download_file(request, BaseSpider('foo')).addCallback(_test)
return self.download_request(request, BaseSpider('foo')).addCallback(_test)
def test_non_existent(self):
request = Request('file://%s' % self.mktemp())
d = download_file(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
return self.assertFailure(d, IOError)
@ -57,6 +60,7 @@ class HttpTestCase(unittest.TestCase):
self.wrapper = WrappingFactory(self.site)
self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1')
self.portno = self.port.getHost().port
self.download_request = HttpRequestHandler().download_request
def tearDown(self):
return self.port.stopListening()
@ -66,28 +70,28 @@ class HttpTestCase(unittest.TestCase):
def test_download(self):
request = Request(self.getURL('file'))
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
d.addCallback(lambda r: r.body)
d.addCallback(self.assertEquals, "0123456789")
return d
def test_download_head(self):
request = Request(self.getURL('file'), method='HEAD')
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
d.addCallback(lambda r: r.body)
d.addCallback(self.assertEquals, '')
return d
def test_redirect_status(self):
request = Request(self.getURL('redirect'))
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
d.addCallback(lambda r: r.status)
d.addCallback(self.assertEquals, 302)
return d
def test_redirect_status_head(self):
request = Request(self.getURL('redirect'), method='HEAD')
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
d.addCallback(lambda r: r.status)
d.addCallback(self.assertEquals, 302)
return d
@ -96,7 +100,7 @@ class HttpTestCase(unittest.TestCase):
spider = BaseSpider('foo')
spider.download_timeout = 0.000001
request = Request(self.getURL('wait'))
d = download_http(request, spider)
d = self.download_request(request, spider)
return self.assertFailure(d, defer.TimeoutError)
def test_host_header_not_in_request_headers(self):
@ -105,7 +109,7 @@ class HttpTestCase(unittest.TestCase):
self.assertEquals(request.headers, {})
request = Request(self.getURL('host'))
return download_http(request, BaseSpider('foo')).addCallback(_test)
return self.download_request(request, BaseSpider('foo')).addCallback(_test)
def test_host_header_seted_in_request_headers(self):
def _test(response):
@ -113,9 +117,9 @@ class HttpTestCase(unittest.TestCase):
self.assertEquals(request.headers.get('Host'), 'example.com')
request = Request(self.getURL('host'), headers={'Host': 'example.com'})
return download_http(request, BaseSpider('foo')).addCallback(_test)
return self.download_request(request, BaseSpider('foo')).addCallback(_test)
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
d.addCallback(lambda r: r.body)
d.addCallback(self.assertEquals, 'example.com')
return d
@ -123,14 +127,14 @@ class HttpTestCase(unittest.TestCase):
def test_payload(self):
body = '1'*100 # PayloadResource requires body length to be 100
request = Request(self.getURL('payload'), method='POST', body=body)
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
d.addCallback(lambda r: r.body)
d.addCallback(self.assertEquals, body)
return d
def test_broken_download(self):
request = Request(self.getURL('broken'))
d = download_http(request, BaseSpider('foo'))
d = self.download_request(request, BaseSpider('foo'))
return self.assertFailure(d, PartialDownloadError)
@ -151,6 +155,7 @@ class HttpProxyTestCase(unittest.TestCase):
wrapper = WrappingFactory(site)
self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')
self.portno = self.port.getHost().port
self.download_request = HttpRequestHandler().download_request
def tearDown(self):
return self.port.stopListening()
@ -166,7 +171,7 @@ class HttpProxyTestCase(unittest.TestCase):
http_proxy = self.getURL('')
request = Request('https://example.com', meta={'proxy': http_proxy})
return download_http(request, BaseSpider('foo')).addCallback(_test)
return self.download_request(request, BaseSpider('foo')).addCallback(_test)
def test_download_without_proxy(self):
def _test(response):
@ -175,4 +180,97 @@ class HttpProxyTestCase(unittest.TestCase):
self.assertEquals(response.body, '/path/to/resource')
request = Request(self.getURL('path/to/resource'))
return download_http(request, BaseSpider('foo')).addCallback(_test)
return self.download_request(request, BaseSpider('foo')).addCallback(_test)
class HttpRequestHandlerMock(object):
def download_request(self, request, spider):
return request
class S3TestCase(unittest.TestCase):
skip = 'boto' not in optional_features and 'missing boto library'
# test use same example keys than amazon developer guide
# http://s3.amazonaws.com/awsdocs/S3/20060301/s3-dg-20060301.pdf
# and the tests described here are the examples from that manual
AWS_ACCESS_KEY_ID = '0PN5J17HBGZHT7JJ3X82'
AWS_SECRET_ACCESS_KEY = 'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
def setUp(self):
s3reqh = S3RequestHandler(self.AWS_ACCESS_KEY_ID, \
self.AWS_SECRET_ACCESS_KEY, \
httprequesthandler=HttpRequestHandlerMock)
self.download_request = s3reqh.download_request
self.spider = BaseSpider('foo')
def test_request_signing1(self):
# gets an object from the johnsmith bucket.
req = Request('s3://johnsmith/photos/puppy.jpg',
headers={'Date': 'Tue, 27 Mar 2007 19:36:42 +0000'})
httpreq = self.download_request(req, self.spider)
self.assertEqual(httpreq.headers['Authorization'], \
'AWS 0PN5J17HBGZHT7JJ3X82:xXjDGYUmKxnwqr5KXNPGldn5LbA=')
def test_request_signing2(self):
# puts an object into the johnsmith bucket.
req = Request('s3://johnsmith/photos/puppy.jpg', method='PUT', headers={
'Content-Type': 'image/jpeg',
'Date': 'Tue, 27 Mar 2007 21:15:45 +0000',
'Content-Length': '94328',
})
httpreq = self.download_request(req, self.spider)
self.assertEqual(httpreq.headers['Authorization'], \
'AWS 0PN5J17HBGZHT7JJ3X82:hcicpDDvL9SsO6AkvxqmIWkmOuQ=')
def test_request_signing3(self):
# lists the content of the johnsmith bucket.
req = Request('s3://johnsmith/?prefix=photos&max-keys=50&marker=puppy', \
method='GET', headers={
'User-Agent': 'Mozilla/5.0',
'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
})
httpreq = self.download_request(req, self.spider)
self.assertEqual(httpreq.headers['Authorization'], \
'AWS 0PN5J17HBGZHT7JJ3X82:jsRt/rhG+Vtp88HrYL706QhE4w4=')
def test_request_signing4(self):
# fetches the access control policy sub-resource for the 'johnsmith' bucket.
req = Request('s3://johnsmith/?acl', \
method='GET', headers={'Date': 'Tue, 27 Mar 2007 19:44:46 +0000'})
httpreq = self.download_request(req, self.spider)
self.assertEqual(httpreq.headers['Authorization'], \
'AWS 0PN5J17HBGZHT7JJ3X82:thdUi9VAkzhkniLj96JIrOPGi0g=')
def test_request_signing5(self):
# deletes an object from the 'johnsmith' bucket using the
# path-style and Date alternative.
req = Request('s3://johnsmith/photos/puppy.jpg', \
method='DELETE', headers={
'Date': 'Tue, 27 Mar 2007 21:20:27 +0000',
'x-amz-date': 'Tue, 27 Mar 2007 21:20:26 +0000',
})
httpreq = self.download_request(req, self.spider)
self.assertEqual(httpreq.headers['Authorization'], \
'AWS 0PN5J17HBGZHT7JJ3X82:k3nL7gH3+PadhTEVn5Ip83xlYzk=')
def test_request_signing6(self):
# uploads an object to a CNAME style virtual hosted bucket with metadata.
req = Request('s3://static.johnsmith.net:8080/db-backup.dat.gz', \
method='PUT', headers={
'User-Agent': 'curl/7.15.5',
'Host': 'static.johnsmith.net:8080',
'Date': 'Tue, 27 Mar 2007 21:06:08 +0000',
'x-amz-acl': 'public-read',
'content-type': 'application/x-download',
'Content-MD5': '4gJE4saaMU4BqNR0kLY+lw==',
'X-Amz-Meta-ReviewedBy': 'joe@johnsmith.net,jane@johnsmith.net',
'X-Amz-Meta-FileChecksum': '0x02661779',
'X-Amz-Meta-ChecksumAlgorithm': 'crc32',
'Content-Disposition': 'attachment; filename=database.dat',
'Content-Encoding': 'gzip',
'Content-Length': '5913339',
})
httpreq = self.download_request(req, self.spider)
self.assertEqual(httpreq.headers['Authorization'], \
'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')