mirror of https://github.com/scrapy/scrapy.git
rewrote of downloader handlers
* add REQUEST_HANDLERS setting with defaults for file, http and https schemes * add documentation of new setting * add unittests for all the builtin handlers * remove unused getPage function
This commit is contained in:
Daniel Grana
parent
40dc867ac3
commit
8aa7d153ae
|
|
@ -725,6 +725,33 @@ Default: ``+2``
|
|||
Adjust redirect request priority relative to original request.
|
||||
A negative priority adjust means more priority.
|
||||
|
||||
.. setting:: REQUEST_HANDLERS
|
||||
|
||||
REQUEST_HANDLERS
|
||||
----------------
|
||||
|
||||
Default: ``{}``
|
||||
|
||||
A dict containing the request downloader handlers enabled in your project.
|
||||
See `REQUEST_HANDLERS_BASE` for example format.
|
||||
|
||||
.. setting:: REQUEST_HANDLERS_BASE
|
||||
|
||||
REQUEST_HANDLERS_BASE
|
||||
---------------------
|
||||
|
||||
Default::
|
||||
|
||||
{
|
||||
'file': 'scrapy.core.downloader.handlers.file.download_file',
|
||||
'http': 'scrapy.core.downloader.handlers.http.download_http',
|
||||
'https': 'scrapy.core.downloader.handlers.http.download_http',
|
||||
}
|
||||
|
||||
A dict containing the request download handlers enabled by default in Scrapy.
|
||||
You should never modify this setting in your project, modify
|
||||
:setting:`REQUEST_HANDLERS` instead.
|
||||
|
||||
.. setting:: REQUESTS_PER_DOMAIN
|
||||
|
||||
REQUESTS_PER_DOMAIN
|
||||
|
|
|
|||
|
|
@ -124,6 +124,13 @@ REDIRECT_MAX_METAREFRESH_DELAY = 100
|
|||
REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
|
||||
REDIRECT_PRIORITY_ADJUST = +2
|
||||
|
||||
REQUEST_HANDLERS = {}
|
||||
REQUEST_HANDLERS_BASE = {
|
||||
'file': 'scrapy.core.downloader.handlers.file.download_file',
|
||||
'http': 'scrapy.core.downloader.handlers.http.download_http',
|
||||
'https': 'scrapy.core.downloader.handlers.http.download_http',
|
||||
}
|
||||
|
||||
REQUESTS_QUEUE_SIZE = 0
|
||||
REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain
|
||||
|
||||
|
|
|
|||
|
|
@ -1,90 +0,0 @@
|
|||
"""
|
||||
Download handlers for different schemes
|
||||
"""
|
||||
from __future__ import with_statement
|
||||
|
||||
import urlparse
|
||||
|
||||
from twisted.internet import reactor
|
||||
try:
|
||||
from twisted.internet import ssl
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
from scrapy import optional_features
|
||||
from scrapy.core import signals
|
||||
from scrapy.http import Headers
|
||||
from scrapy.core.exceptions import NotSupported
|
||||
from scrapy.utils.defer import defer_succeed
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.signal import send_catch_log
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.core.downloader.responsetypes import responsetypes
|
||||
from scrapy.conf import settings
|
||||
|
||||
|
||||
HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
|
||||
default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
|
||||
ssl_supported = 'ssl' in optional_features
|
||||
|
||||
def download_any(request, spider):
|
||||
scheme = urlparse_cached(request).scheme
|
||||
if scheme == 'http':
|
||||
return download_http(request, spider)
|
||||
elif scheme == 'https':
|
||||
if ssl_supported:
|
||||
return download_https(request, spider)
|
||||
else:
|
||||
raise NotSupported("HTTPS not supported: install pyopenssl library")
|
||||
elif scheme == 'file':
|
||||
return download_file(request, spider)
|
||||
else:
|
||||
raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url))
|
||||
|
||||
def create_factory(request, spider):
|
||||
"""Return HTTPClientFactory for the given Request"""
|
||||
url = urlparse.urldefrag(request.url)[0]
|
||||
timeout = getattr(spider, "download_timeout", None) or default_timeout
|
||||
factory = HTTPClientFactory.from_request(request, timeout)
|
||||
|
||||
def _create_response(body):
|
||||
body = body or ''
|
||||
status = int(factory.status)
|
||||
headers = Headers(factory.response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=url)
|
||||
r = respcls(url=request.url, status=status, headers=headers, body=body)
|
||||
send_catch_log(signal=signals.request_uploaded, sender='download_http', \
|
||||
request=request, spider=spider)
|
||||
send_catch_log(signal=signals.response_downloaded, sender='download_http', \
|
||||
response=r, spider=spider)
|
||||
return r
|
||||
|
||||
factory.deferred.addCallbacks(_create_response)
|
||||
return factory
|
||||
|
||||
def download_http(request, spider):
|
||||
"""Return a deferred for the HTTP download"""
|
||||
factory = create_factory(request, spider)
|
||||
url = urlparse_cached(request)
|
||||
port = url.port
|
||||
reactor.connectTCP(url.hostname, port or 80, factory)
|
||||
return factory.deferred
|
||||
|
||||
def download_https(request, spider):
|
||||
"""Return a deferred for the HTTPS download"""
|
||||
factory = create_factory(request, spider)
|
||||
url = urlparse_cached(request)
|
||||
port = url.port
|
||||
contextFactory = ssl.ClientContextFactory()
|
||||
reactor.connectSSL(url.hostname, port or 443, factory, contextFactory)
|
||||
return factory.deferred
|
||||
|
||||
def download_file(request, spider) :
|
||||
"""Return a deferred for a file download."""
|
||||
filepath = request.url.split("file://")[1]
|
||||
with open(filepath) as f:
|
||||
body = f.read()
|
||||
respcls = responsetypes.from_args(filename=filepath, body=body)
|
||||
response = respcls(url=request.url, body=body)
|
||||
|
||||
return defer_succeed(response)
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
"""Download handlers for different schemes"""
|
||||
|
||||
from scrapy.core.exceptions import NotSupported
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.conf import settings
|
||||
from scrapy.utils.misc import load_object
|
||||
|
||||
|
||||
class RequestHandlers(object):
|
||||
|
||||
def __init__(self):
|
||||
self._handlers = {}
|
||||
handlers = settings.get('REQUEST_HANDLERS_BASE')
|
||||
handlers.update(settings.get('REQUEST_HANDLERS', {}))
|
||||
for scheme, cls in handlers.iteritems():
|
||||
self._handlers[scheme] = load_object(cls)
|
||||
|
||||
def download_request(self, request, spider):
|
||||
scheme = urlparse_cached(request).scheme
|
||||
try:
|
||||
handler = self._handlers[scheme]
|
||||
except KeyError:
|
||||
raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url))
|
||||
return handler(request, spider)
|
||||
|
||||
|
||||
download_any = RequestHandlers().download_request
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
"""Download handler for file:// scheme"""
|
||||
from __future__ import with_statement
|
||||
|
||||
from twisted.internet import defer
|
||||
from scrapy.core.downloader.responsetypes import responsetypes
|
||||
|
||||
|
||||
def download_file(request, spider):
|
||||
"""Return a deferred for a file download."""
|
||||
return defer.maybeDeferred(_all_in_one_read_download_file, request, spider)
|
||||
|
||||
def _all_in_one_read_download_file(request, spider):
|
||||
filepath = request.url.split("file://")[1]
|
||||
with open(filepath) as f:
|
||||
body = f.read()
|
||||
respcls = responsetypes.from_args(filename=filepath, body=body)
|
||||
return respcls(url=request.url, body=body)
|
||||
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
"""Download handlers for http and https schemes"""
|
||||
|
||||
from twisted.internet import reactor
|
||||
|
||||
from scrapy.core import signals
|
||||
from scrapy.core.exceptions import NotSupported
|
||||
from scrapy.utils.signal import send_catch_log
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.conf import settings
|
||||
from scrapy import optional_features
|
||||
|
||||
ssl_supported = 'ssl' in optional_features
|
||||
if ssl_supported:
|
||||
from twisted.internet.ssl import ClientContextFactory
|
||||
|
||||
|
||||
HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
|
||||
default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
|
||||
|
||||
def _create_factory(request, spider):
|
||||
def _download_signals(response):
|
||||
send_catch_log(signal=signals.request_uploaded, \
|
||||
sender='download_http', request=request, spider=spider)
|
||||
send_catch_log(signal=signals.response_downloaded, \
|
||||
sender='download_http', response=response, spider=spider)
|
||||
return response
|
||||
|
||||
timeout = getattr(spider, "download_timeout", None) or default_timeout
|
||||
factory = HTTPClientFactory(request, timeout)
|
||||
factory.deferred.addCallbacks(_download_signals)
|
||||
return factory
|
||||
|
||||
|
||||
def _connect(factory):
|
||||
host, port = factory.host, factory.port
|
||||
if factory.scheme == 'https':
|
||||
if ssl_supported:
|
||||
return reactor.connectSSL(host, port, factory, ClientContextFactory())
|
||||
raise NotSupported("HTTPS not supported: install pyopenssl library")
|
||||
else:
|
||||
return reactor.connectTCP(host, port, factory)
|
||||
|
||||
|
||||
def download_http(request, spider):
|
||||
"""Return a deferred for the HTTP download"""
|
||||
factory = _create_factory(request, spider)
|
||||
_connect(factory)
|
||||
return factory.deferred
|
||||
|
||||
|
||||
|
|
@ -1,12 +1,14 @@
|
|||
from urlparse import urlparse, urlunparse
|
||||
from urlparse import urlparse, urlunparse, urldefrag
|
||||
|
||||
from twisted.python import failure
|
||||
from twisted.web.client import HTTPClientFactory, PartialDownloadError
|
||||
from twisted.web.client import PartialDownloadError, HTTPClientFactory
|
||||
from twisted.web.http import HTTPClient
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy.http import Headers
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.core.downloader.responsetypes import responsetypes
|
||||
|
||||
|
||||
def _parsed_url_args(parsed):
|
||||
path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
|
||||
|
|
@ -18,11 +20,13 @@ def _parsed_url_args(parsed):
|
|||
port = 443 if scheme == 'https' else 80
|
||||
return scheme, netloc, host, port, path
|
||||
|
||||
|
||||
def _parse(url):
|
||||
url = url.strip()
|
||||
parsed = urlparse(url)
|
||||
return _parsed_url_args(parsed)
|
||||
|
||||
|
||||
class ScrapyHTTPPageGetter(HTTPClient):
|
||||
|
||||
def connectionMade(self):
|
||||
|
|
@ -64,7 +68,9 @@ class ScrapyHTTPPageGetter(HTTPClient):
|
|||
|
||||
def timeout(self):
|
||||
self.transport.loseConnection()
|
||||
self.factory.noPage(defer.TimeoutError("Getting %s took longer than %s seconds." % (self.factory.url, self.factory.timeout)))
|
||||
self.factory.noPage(\
|
||||
defer.TimeoutError("Getting %s took longer than %s seconds." % \
|
||||
(self.factory.url, self.factory.timeout)))
|
||||
|
||||
|
||||
class ScrapyHTTPClientFactory(HTTPClientFactory):
|
||||
|
|
@ -74,22 +80,19 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
|
|||
"""
|
||||
|
||||
protocol = ScrapyHTTPPageGetter
|
||||
response_headers = None
|
||||
waiting = 1
|
||||
noisy = False
|
||||
|
||||
def __init__(self, url, method='GET', body=None, headers=None, timeout=0, parsedurl=None):
|
||||
self.url = url
|
||||
self.method = method
|
||||
self.body = body or None
|
||||
if parsedurl:
|
||||
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsedurl)
|
||||
else:
|
||||
self.scheme, self.netloc, self.host, self.port, self.path = _parse(url)
|
||||
|
||||
def __init__(self, request, timeout=0):
|
||||
self.url = urldefrag(request.url)[0]
|
||||
self.method = request.method
|
||||
self.body = request.body or None
|
||||
self.headers = request.headers
|
||||
self.response_headers = None
|
||||
self.timeout = timeout
|
||||
self.headers = Headers(headers or {})
|
||||
self.deferred = defer.Deferred()
|
||||
self.deferred = defer.Deferred().addCallback(self._build_response)
|
||||
|
||||
self._set_connection_attributes(request)
|
||||
|
||||
# set Host header based on url
|
||||
self.headers.setdefault('Host', self.netloc)
|
||||
|
|
@ -100,33 +103,15 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
|
|||
# just in case a broken http/1.1 decides to keep connection alive
|
||||
self.headers.setdefault("Connection", "close")
|
||||
|
||||
@classmethod
|
||||
def from_request(cls, request, timeout):
|
||||
return cls(request.url,
|
||||
method=request.method,
|
||||
body=request.body or None, # see http://dev.scrapy.org/ticket/60
|
||||
headers=Headers(request.headers or {}),
|
||||
timeout=timeout,
|
||||
parsedurl=urlparse_cached(request),
|
||||
)
|
||||
def _build_response(self, body):
|
||||
status = int(self.status)
|
||||
headers = Headers(self.response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=self.url)
|
||||
return respcls(url=self.url, status=status, headers=headers, body=body)
|
||||
|
||||
def _set_connection_attributes(self, request):
|
||||
parsed = urlparse_cached(request)
|
||||
self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
|
||||
|
||||
def gotHeaders(self, headers):
|
||||
self.response_headers = headers
|
||||
|
||||
|
||||
|
||||
def getPage(url, contextFactory=None, *args, **kwargs):
|
||||
"""
|
||||
Download a web page as a string.
|
||||
|
||||
Download a page. Return a deferred, which will callback with a
|
||||
page (as a string) or errback with a description of the error.
|
||||
|
||||
See HTTPClientFactory to see what extra args can be passed.
|
||||
"""
|
||||
from twisted.web.client import _makeGetterFactory
|
||||
return _makeGetterFactory(
|
||||
url,
|
||||
ScrapyHTTPClientFactory,
|
||||
contextFactory=contextFactory,
|
||||
*args, **kwargs).deferred
|
||||
|
|
|
|||
|
|
@ -0,0 +1,105 @@
|
|||
import os
|
||||
|
||||
from twisted.trial import unittest
|
||||
from twisted.protocols.policies import WrappingFactory
|
||||
from twisted.python.filepath import FilePath
|
||||
from twisted.internet import reactor, defer
|
||||
from twisted.web import server, static, util, resource
|
||||
from twisted.web.test.test_webclient import ForeverTakingResource, \
|
||||
NoLengthResource, HostHeaderResource, \
|
||||
PayloadResource, BrokenDownloadResource
|
||||
|
||||
from scrapy.core.downloader.webclient import PartialDownloadError
|
||||
from scrapy.core.downloader.handlers.file import download_file
|
||||
from scrapy.core.downloader.handlers.http import download_http
|
||||
from scrapy.spider import BaseSpider
|
||||
from scrapy.http import Request
|
||||
|
||||
|
||||
class FileTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.tmpname = self.mktemp()
|
||||
fd = open(self.tmpname, 'w')
|
||||
fd.write('0123456789')
|
||||
fd.close()
|
||||
|
||||
def test_download(self):
|
||||
def _test(response):
|
||||
self.assertEquals(response.url, request.url)
|
||||
self.assertEquals(response.status, 200)
|
||||
self.assertEquals(response.body, '0123456789')
|
||||
|
||||
request = Request('file://%s' % self.tmpname)
|
||||
return download_file(request, BaseSpider()).addCallback(_test)
|
||||
|
||||
def test_non_existent(self):
|
||||
request = Request('file://%s' % self.mktemp())
|
||||
d = download_file(request, BaseSpider())
|
||||
return self.assertFailure(d, IOError)
|
||||
|
||||
|
||||
class HttpTestCase(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
name = self.mktemp()
|
||||
os.mkdir(name)
|
||||
FilePath(name).child("file").setContent("0123456789")
|
||||
r = static.File(name)
|
||||
r.putChild("redirect", util.Redirect("/file"))
|
||||
r.putChild("wait", ForeverTakingResource())
|
||||
r.putChild("nolength", NoLengthResource())
|
||||
r.putChild("host", HostHeaderResource())
|
||||
r.putChild("payload", PayloadResource())
|
||||
r.putChild("broken", BrokenDownloadResource())
|
||||
self.site = server.Site(r, timeout=None)
|
||||
self.wrapper = WrappingFactory(self.site)
|
||||
self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1')
|
||||
self.portno = self.port.getHost().port
|
||||
|
||||
def tearDown(self):
|
||||
return self.port.stopListening()
|
||||
|
||||
def getURL(self, path):
|
||||
return "http://127.0.0.1:%d/%s" % (self.portno, path)
|
||||
|
||||
def test_download(self):
|
||||
request = Request(self.getURL('file'))
|
||||
d = download_http(request, BaseSpider())
|
||||
d.addCallback(lambda r: r.body)
|
||||
d.addCallback(self.assertEquals, "0123456789")
|
||||
return d
|
||||
|
||||
def test_redirect_status(self):
|
||||
request = Request(self.getURL('redirect'))
|
||||
d = download_http(request, BaseSpider())
|
||||
d.addCallback(lambda r: r.status)
|
||||
d.addCallback(self.assertEquals, 302)
|
||||
return d
|
||||
|
||||
def test_timeout_download_from_spider(self):
|
||||
spider = BaseSpider()
|
||||
spider.download_timeout = 0.000001
|
||||
request = Request(self.getURL('wait'))
|
||||
d = download_http(request, spider)
|
||||
return self.assertFailure(d, defer.TimeoutError)
|
||||
|
||||
def test_host_header(self):
|
||||
request = Request(self.getURL('host'))
|
||||
d = download_http(request, BaseSpider())
|
||||
d.addCallback(lambda r: r.body)
|
||||
d.addCallback(self.assertEquals, '127.0.0.1:%d' % self.portno)
|
||||
return d
|
||||
|
||||
def test_payload(self):
|
||||
body = '1'*100 # PayloadResource requires body length to be 100
|
||||
request = Request(self.getURL('payload'), method='POST', body=body)
|
||||
d = download_http(request, BaseSpider())
|
||||
d.addCallback(lambda r: r.body)
|
||||
d.addCallback(self.assertEquals, body)
|
||||
return d
|
||||
|
||||
def test_broken_download(self):
|
||||
request = Request(self.getURL('broken'))
|
||||
d = download_http(request, BaseSpider())
|
||||
return self.assertFailure(d, PartialDownloadError)
|
||||
|
|
@ -13,14 +13,27 @@ from twisted.python.filepath import FilePath
|
|||
from twisted.protocols.policies import WrappingFactory
|
||||
|
||||
from scrapy.core.downloader import webclient as client
|
||||
from scrapy.http import Headers
|
||||
from scrapy.http import Request, Headers
|
||||
|
||||
|
||||
def getPage(url, contextFactory=None, *args, **kwargs):
|
||||
"""Adapted version of twisted.web.client.getPage"""
|
||||
def _clientfactory(*args, **kwargs):
|
||||
timeout = kwargs.pop('timeout', 0)
|
||||
f = client.ScrapyHTTPClientFactory(Request(*args, **kwargs), timeout=timeout)
|
||||
f.deferred.addCallback(lambda r: r.body)
|
||||
return f
|
||||
|
||||
from twisted.web.client import _makeGetterFactory
|
||||
return _makeGetterFactory(url, _clientfactory,
|
||||
contextFactory=contextFactory, *args, **kwargs).deferred
|
||||
|
||||
|
||||
class ParseUrlTestCase(unittest.TestCase):
|
||||
"""Test URL parsing facility and defaults values."""
|
||||
|
||||
def _parse(self, url):
|
||||
f = client.ScrapyHTTPClientFactory(url)
|
||||
f = client.ScrapyHTTPClientFactory(Request(url))
|
||||
return (f.scheme, f.netloc, f.host, f.port, f.path)
|
||||
|
||||
def testParse(self):
|
||||
|
|
@ -75,7 +88,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
|
||||
def test_earlyHeaders(self):
|
||||
# basic test stolen from twisted HTTPageGetter
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
factory = client.ScrapyHTTPClientFactory(Request(
|
||||
url='http://foo/bar',
|
||||
body="some data",
|
||||
headers={
|
||||
|
|
@ -83,7 +96,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
'User-Agent': 'fooble',
|
||||
'Cookie': 'blah blah',
|
||||
'Content-Length': '12981',
|
||||
'Useful': 'value'})
|
||||
'Useful': 'value'}))
|
||||
|
||||
self._test(factory,
|
||||
"GET /bar HTTP/1.0\r\n"
|
||||
|
|
@ -97,18 +110,18 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
"some data")
|
||||
|
||||
# test minimal sent headers
|
||||
factory = client.ScrapyHTTPClientFactory('http://foo/bar')
|
||||
factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar'))
|
||||
self._test(factory,
|
||||
"GET /bar HTTP/1.0\r\n"
|
||||
"Host: foo\r\n"
|
||||
"\r\n")
|
||||
|
||||
# test a simple POST with body and content-type
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
factory = client.ScrapyHTTPClientFactory(Request(
|
||||
method='POST',
|
||||
url='http://foo/bar',
|
||||
body='name=value',
|
||||
headers={'Content-Type': 'application/x-www-form-urlencoded'})
|
||||
headers={'Content-Type': 'application/x-www-form-urlencoded'}))
|
||||
|
||||
self._test(factory,
|
||||
"POST /bar HTTP/1.0\r\n"
|
||||
|
|
@ -120,12 +133,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
"name=value")
|
||||
|
||||
# test with single and multivalued headers
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
factory = client.ScrapyHTTPClientFactory(Request(
|
||||
url='http://foo/bar',
|
||||
headers={
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
})
|
||||
}))
|
||||
|
||||
self._test(factory,
|
||||
"GET /bar HTTP/1.0\r\n"
|
||||
|
|
@ -136,12 +149,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
|||
"\r\n")
|
||||
|
||||
# same test with single and multivalued headers but using Headers class
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
factory = client.ScrapyHTTPClientFactory(Request(
|
||||
url='http://foo/bar',
|
||||
headers=Headers({
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
}))
|
||||
})))
|
||||
|
||||
self._test(factory,
|
||||
"GET /bar HTTP/1.0\r\n"
|
||||
|
|
@ -193,11 +206,11 @@ class WebClientTestCase(unittest.TestCase):
|
|||
|
||||
def testPayload(self):
|
||||
s = "0123456789" * 10
|
||||
return client.getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s)
|
||||
return getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s)
|
||||
|
||||
def testBrokenDownload(self):
|
||||
# test what happens when download gets disconnected in the middle
|
||||
d = client.getPage(self.getURL("broken"))
|
||||
d = getPage(self.getURL("broken"))
|
||||
d = self.assertFailure(d, client.PartialDownloadError)
|
||||
d.addCallback(lambda exc: self.assertEquals(exc.response, "abc"))
|
||||
return d
|
||||
|
|
@ -206,8 +219,8 @@ class WebClientTestCase(unittest.TestCase):
|
|||
# if we pass Host header explicitly, it should be used, otherwise
|
||||
# it should extract from url
|
||||
return defer.gatherResults([
|
||||
client.getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno),
|
||||
client.getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")])
|
||||
getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno),
|
||||
getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")])
|
||||
|
||||
|
||||
def test_getPage(self):
|
||||
|
|
@ -215,7 +228,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
L{client.getPage} returns a L{Deferred} which is called back with
|
||||
the body of the response if the default method B{GET} is used.
|
||||
"""
|
||||
d = client.getPage(self.getURL("file"))
|
||||
d = getPage(self.getURL("file"))
|
||||
d.addCallback(self.assertEquals, "0123456789")
|
||||
return d
|
||||
|
||||
|
|
@ -226,11 +239,11 @@ class WebClientTestCase(unittest.TestCase):
|
|||
the empty string if the method is C{HEAD} and there is a successful
|
||||
response code.
|
||||
"""
|
||||
def getPage(method):
|
||||
return client.getPage(self.getURL("file"), method=method)
|
||||
def _getPage(method):
|
||||
return getPage(self.getURL("file"), method=method)
|
||||
return defer.gatherResults([
|
||||
getPage("head").addCallback(self.assertEqual, ""),
|
||||
getPage("HEAD").addCallback(self.assertEqual, "")])
|
||||
_getPage("head").addCallback(self.assertEqual, ""),
|
||||
_getPage("HEAD").addCallback(self.assertEqual, "")])
|
||||
|
||||
|
||||
def test_timeoutNotTriggering(self):
|
||||
|
|
@ -239,7 +252,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
retrieved before the timeout period elapses, the L{Deferred} is
|
||||
called back with the contents of the page.
|
||||
"""
|
||||
d = client.getPage(self.getURL("host"), timeout=100)
|
||||
d = getPage(self.getURL("host"), timeout=100)
|
||||
d.addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno)
|
||||
return d
|
||||
|
||||
|
|
@ -251,7 +264,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
L{Deferred} is errbacked with a L{error.TimeoutError}.
|
||||
"""
|
||||
finished = self.assertFailure(
|
||||
client.getPage(self.getURL("wait"), timeout=0.000001),
|
||||
getPage(self.getURL("wait"), timeout=0.000001),
|
||||
defer.TimeoutError)
|
||||
def cleanup(passthrough):
|
||||
# Clean up the server which is hanging around not doing
|
||||
|
|
@ -266,7 +279,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
return finished
|
||||
|
||||
def testNotFound(self):
|
||||
return client.getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile)
|
||||
return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile)
|
||||
|
||||
def _cbNoSuchFile(self, pageData):
|
||||
self.assert_('404 - No Such Resource' in pageData)
|
||||
|
|
@ -274,7 +287,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
def testFactoryInfo(self):
|
||||
url = self.getURL('file')
|
||||
scheme, netloc, host, port, path = client._parse(url)
|
||||
factory = client.ScrapyHTTPClientFactory(url)
|
||||
factory = client.ScrapyHTTPClientFactory(Request(url))
|
||||
reactor.connectTCP(host, port, factory)
|
||||
return factory.deferred.addCallback(self._cbFactoryInfo, factory)
|
||||
|
||||
|
|
@ -285,7 +298,7 @@ class WebClientTestCase(unittest.TestCase):
|
|||
self.assertEquals(factory.response_headers['content-length'], '10')
|
||||
|
||||
def testRedirect(self):
|
||||
return client.getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
|
||||
return getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
|
||||
|
||||
def _cbRedirect(self, pageData):
|
||||
self.assertEquals(pageData,
|
||||
|
|
|
|||
Loading…
Reference in New Issue