mirror of https://github.com/scrapy/scrapy.git
core: clean HTTP handling and add unittests
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401089
This commit is contained in:
Daniel Grana
parent
0842fafd63
commit
e8ecc00cfb
|
|
@ -25,7 +25,6 @@ from scrapy.core.downloader.responsetypes import responsetypes
|
|||
from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory as HTTPClientFactory
|
||||
|
||||
default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
|
||||
default_agent = settings.get('USER_AGENT')
|
||||
ssl_supported = 'ssl' in optional_features
|
||||
|
||||
# Cache for dns lookups.
|
||||
|
|
@ -48,15 +47,13 @@ def download_any(request, spider):
|
|||
def create_factory(request, spider):
|
||||
"""Return HTTPClientFactory for the given Request"""
|
||||
url = urlparse.urldefrag(request.url)[0]
|
||||
timeout = getattr(spider, "download_timeout", None) or default_timeout
|
||||
|
||||
agent = request.headers.pop('user-agent', default_agent)
|
||||
factory = HTTPClientFactory(url=url, # never pass unicode urls to twisted
|
||||
method=request.method,
|
||||
postdata=request.body or None, # see http://dev.scrapy.org/ticket/60
|
||||
body=request.body or None, # see http://dev.scrapy.org/ticket/60
|
||||
headers=request.headers,
|
||||
agent=agent,
|
||||
timeout=getattr(spider, "download_timeout", None) or default_timeout,
|
||||
followRedirect=False)
|
||||
timeout=timeout)
|
||||
|
||||
def _create_response(body):
|
||||
body = body or ''
|
||||
|
|
@ -69,16 +66,12 @@ def create_factory(request, spider):
|
|||
return r
|
||||
|
||||
def _on_success(body):
|
||||
return _create_response(body)
|
||||
response = _create_response(body)
|
||||
if response.status not in (200, 201, 202):
|
||||
raise HttpException(response.status, None, response)
|
||||
return response
|
||||
|
||||
def _on_error(_failure):
|
||||
ex = _failure.value
|
||||
if isinstance(ex, web_error.Error): # HttpException
|
||||
raise HttpException(ex.status, ex.message, _create_response(ex.response))
|
||||
return _failure
|
||||
|
||||
factory.noisy = False
|
||||
factory.deferred.addCallbacks(_on_success, _on_error)
|
||||
factory.deferred.addCallbacks(_on_success)
|
||||
return factory
|
||||
|
||||
def download_http(request, spider):
|
||||
|
|
|
|||
|
|
@ -1,152 +1,121 @@
|
|||
from urlparse import urlunparse
|
||||
|
||||
from twisted.web.client import HTTPClientFactory, HTTPPageGetter
|
||||
from twisted.web import http
|
||||
from twisted.python import failure
|
||||
from twisted.web import error
|
||||
from twisted.web.client import HTTPClientFactory, PartialDownloadError
|
||||
from twisted.web.http import HTTPClient
|
||||
from twisted.internet import defer
|
||||
|
||||
from scrapy.http import Url, Headers
|
||||
from scrapy.utils.misc import arg_to_iter
|
||||
|
||||
def _parse(url, defaultPort=None):
|
||||
|
||||
def _parse(url):
|
||||
url = url.strip()
|
||||
try:
|
||||
parsed = url.parsedurl
|
||||
except AttributeError:
|
||||
parsed = Url(url).parsedurl
|
||||
parsed = Url(url.strip()).parsedurl
|
||||
|
||||
path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
|
||||
host = parsed.hostname
|
||||
port = parsed.port
|
||||
scheme = parsed.scheme
|
||||
|
||||
if port is None:
|
||||
port = 443 if scheme == 'https' else 80
|
||||
|
||||
scheme = parsed[0]
|
||||
path = urlunparse(('','')+parsed[2:])
|
||||
if defaultPort is None:
|
||||
if scheme == 'https':
|
||||
defaultPort = 443
|
||||
else:
|
||||
defaultPort = 80
|
||||
host, port = parsed[1], defaultPort
|
||||
if ':' in host:
|
||||
host, port = host.split(':')
|
||||
port = int(port)
|
||||
if path == "":
|
||||
path = "/"
|
||||
return scheme, host, port, path
|
||||
|
||||
|
||||
class ScrapyHTTPPageGetter(HTTPPageGetter):
|
||||
quietLoss = 0
|
||||
failed = 0
|
||||
|
||||
_specialHeaders = set(('host', 'user-agent', 'content-length'))
|
||||
class ScrapyHTTPPageGetter(HTTPClient):
|
||||
|
||||
def connectionMade(self):
|
||||
headers = self.factory.headers
|
||||
method = getattr(self.factory, 'method', 'GET')
|
||||
self.headers = Headers() # bucket for response headers
|
||||
|
||||
self.sendCommand(method, self.factory.path)
|
||||
self.sendHeader('Host', headers.get("host", self.factory.host))
|
||||
self.sendHeader('User-Agent', headers.get('User-Agent', self.factory.agent))
|
||||
|
||||
data = getattr(self.factory, 'postdata', None)
|
||||
if data is not None:
|
||||
self.sendHeader("Content-Length", str(len(data)))
|
||||
|
||||
for key, value in self.factory.headers.items():
|
||||
if key.lower() not in self._specialHeaders:
|
||||
# Method command
|
||||
self.sendCommand(self.factory.method, self.factory.path)
|
||||
# Headers
|
||||
for key, values in self.factory.headers.items():
|
||||
for value in values:
|
||||
self.sendHeader(key, value)
|
||||
|
||||
self.endHeaders()
|
||||
self.headers = Headers()
|
||||
|
||||
if data is not None:
|
||||
self.transport.write(data)
|
||||
|
||||
def sendHeader(self, name, value):
|
||||
for v in arg_to_iter(value):
|
||||
self.transport.write('%s: %s\r\n' % (name, v))
|
||||
# Body
|
||||
if self.factory.body is not None:
|
||||
self.transport.write(self.factory.body)
|
||||
|
||||
def handleHeader(self, key, value):
|
||||
self.headers.appendlist(key, value)
|
||||
|
||||
def handleStatus(self, version, status, message):
|
||||
self.version, self.status, self.message = version, status, message
|
||||
self.factory.gotStatus(version, status, message)
|
||||
|
||||
def handleEndHeaders(self):
|
||||
self.factory.gotHeaders(self.headers)
|
||||
m = getattr(self, 'handleStatus_'+self.status, self.handleStatusDefault)
|
||||
m()
|
||||
|
||||
def handleStatus_200(self):
|
||||
pass
|
||||
|
||||
handleStatus_201 = lambda self: self.handleStatus_200()
|
||||
handleStatus_202 = lambda self: self.handleStatus_200()
|
||||
|
||||
def handleStatusDefault(self):
|
||||
self.failed = 1
|
||||
|
||||
def connectionLost(self, reason):
|
||||
if not self.quietLoss:
|
||||
http.HTTPClient.connectionLost(self, reason)
|
||||
self.factory.noPage(reason)
|
||||
HTTPClient.connectionLost(self, reason)
|
||||
self.factory.noPage(reason)
|
||||
|
||||
def handleResponse(self, response):
|
||||
if self.quietLoss:
|
||||
return
|
||||
|
||||
if self.failed:
|
||||
self.factory.noPage(failure.Failure(error.Error(self.status, self.message, response)))
|
||||
|
||||
if self.factory.method.upper() == 'HEAD':
|
||||
# Callback with empty string, since there is never a response
|
||||
# body for HEAD requests.
|
||||
self.factory.page('')
|
||||
elif self.length != None and self.length != 0:
|
||||
self.factory.noPage(failure.Failure(
|
||||
PartialDownloadError(self.status, self.message, response)))
|
||||
PartialDownloadError(self.factory.status, None, response)))
|
||||
else:
|
||||
self.factory.page(response)
|
||||
|
||||
# server might be stupid and not close connection. admittedly
|
||||
# the fact we do only one request per connection is also
|
||||
# stupid...
|
||||
self.transport.loseConnection()
|
||||
|
||||
def timeout(self):
|
||||
self.quietLoss = True
|
||||
self.transport.loseConnection()
|
||||
self.factory.noPage(defer.TimeoutError("Getting %s took longer than %s seconds." % (self.factory.url, self.factory.timeout)))
|
||||
|
||||
|
||||
|
||||
class ScrapyHTTPClientFactory(HTTPClientFactory):
|
||||
"""Scrapy implementation of the HTTPClientFactory overwriting the
|
||||
serUrl method to make use of our Url object that cache the parse
|
||||
result. Also we override gotHeaders that dies when parsing malformed
|
||||
cookies.
|
||||
result.
|
||||
"""
|
||||
|
||||
protocol = ScrapyHTTPPageGetter
|
||||
response_headers = None
|
||||
waiting = 1
|
||||
noisy = False
|
||||
|
||||
def setURL(self, url):
|
||||
def __init__(self, url, method='GET', body=None, headers=None, timeout=0):
|
||||
self.url = url
|
||||
scheme, host, port, path = _parse(url)
|
||||
if scheme and host:
|
||||
self.scheme = scheme
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.path = path
|
||||
self.method = method
|
||||
self.body = body or None
|
||||
self.scheme, self.host, self.port, self.path = _parse(url)
|
||||
self.timeout = timeout
|
||||
self.headers = Headers(headers or {})
|
||||
self.deferred = defer.Deferred()
|
||||
|
||||
# set Host header based on url
|
||||
self.headers.setdefault('Host', self.host)
|
||||
|
||||
# set Content-Length based len of body
|
||||
if self.body is not None:
|
||||
self.headers['Content-Length'] = len(self.body)
|
||||
# just in case a broken http/1.1 decides to keep connection alive
|
||||
self.headers.setdefault("Connection", "close")
|
||||
|
||||
def gotHeaders(self, headers):
|
||||
"""
|
||||
HTTPClientFactory.gotHeaders dies when parsing malformed cookies,
|
||||
and the crawler is getting malformed cookies from this site.
|
||||
|
||||
Cookies format:
|
||||
http://www.ietf.org/rfc/rfc2109.txt
|
||||
|
||||
I have choosen not to filter based on this, so we don't filter invalid
|
||||
values that could be managed correctly by twisted.
|
||||
"""
|
||||
self.response_headers = headers
|
||||
|
||||
|
||||
|
||||
def getPage(url, contextFactory=None, *args, **kwargs):
|
||||
"""
|
||||
Download a web page as a string.
|
||||
|
||||
Download a page. Return a deferred, which will callback with a
|
||||
page (as a string) or errback with a description of the error.
|
||||
|
||||
See HTTPClientFactory to see what extra args can be passed.
|
||||
"""
|
||||
from twisted.web.client import _makeGetterFactory
|
||||
return _makeGetterFactory(
|
||||
url,
|
||||
ScrapyHTTPClientFactory,
|
||||
contextFactory=contextFactory,
|
||||
*args, **kwargs).deferred
|
||||
|
|
|
|||
|
|
@ -2,41 +2,55 @@
|
|||
from twisted.internet import defer
|
||||
Tests borrowed from the twisted.web.client tests.
|
||||
"""
|
||||
import os
|
||||
from urlparse import urlparse
|
||||
|
||||
from twisted.trial import unittest
|
||||
from twisted.web import server, static
|
||||
from twisted.web import server, static, error, util
|
||||
from twisted.internet import reactor, defer
|
||||
from twisted.test.proto_helpers import StringTransport
|
||||
from twisted.python.filepath import FilePath
|
||||
from twisted.protocols.policies import WrappingFactory
|
||||
|
||||
from scrapy.core.downloader import webclient as client
|
||||
from scrapy.http import Url, Headers
|
||||
|
||||
from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
|
||||
from scrapy.http import Url
|
||||
|
||||
class ParseUrlTestCase(unittest.TestCase):
|
||||
"""Test URL parsing facility and defaults values."""
|
||||
|
||||
def _parse(self, url):
|
||||
f = ScrapyHTTPClientFactory(Url(url))
|
||||
f = client.ScrapyHTTPClientFactory(Url(url))
|
||||
return (f.scheme, f.host, f.port, f.path)
|
||||
|
||||
def testParse(self):
|
||||
scheme, host, port, path = self._parse("http://127.0.0.1/?param=value")
|
||||
self.assertEquals(path, "/?param=value")
|
||||
self.assertEquals(port, 80)
|
||||
scheme, host, port, path = self._parse("http://127.0.0.1/")
|
||||
self.assertEquals(path, "/")
|
||||
self.assertEquals(port, 80)
|
||||
scheme, host, port, path = self._parse("https://127.0.0.1/")
|
||||
self.assertEquals(path, "/")
|
||||
self.assertEquals(port, 443)
|
||||
scheme, host, port, path = self._parse("http://spam:12345/")
|
||||
self.assertEquals(port, 12345)
|
||||
scheme, host, port, path = self._parse("http://foo ")
|
||||
self.assertEquals(host, "foo")
|
||||
self.assertEquals(path, "/")
|
||||
scheme, host, port, path = self._parse("http://egg:7890")
|
||||
self.assertEquals(port, 7890)
|
||||
self.assertEquals(host, "egg")
|
||||
self.assertEquals(path, "/")
|
||||
tests = (
|
||||
("http://127.0.0.1?c=v&c2=v2#fragment", ('http', '127.0.0.1', 80, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', '127.0.0.1', 80, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', '127.0.0.1', 80, '/foo?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', '127.0.0.1', 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', '127.0.0.1', 100, '/?c=v&c2=v2')),
|
||||
("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', '127.0.0.1', 100, '/foo?c=v&c2=v2')),
|
||||
|
||||
("http://127.0.0.1", ('http', '127.0.0.1', 80, '/')),
|
||||
("http://127.0.0.1/", ('http', '127.0.0.1', 80, '/')),
|
||||
("http://127.0.0.1/foo", ('http', '127.0.0.1', 80, '/foo')),
|
||||
("http://127.0.0.1?param=value", ('http', '127.0.0.1', 80, '/?param=value')),
|
||||
("http://127.0.0.1/?param=value", ('http', '127.0.0.1', 80, '/?param=value')),
|
||||
("http://127.0.0.1:12345/foo", ('http', '127.0.0.1', 12345, '/foo')),
|
||||
("http://spam:12345/foo", ('http', 'spam', 12345, '/foo')),
|
||||
("http://spam.scrapytest.org/foo", ('http', 'spam.scrapytest.org', 80, '/foo')),
|
||||
|
||||
("https://127.0.0.1/foo", ('https', '127.0.0.1', 443, '/foo')),
|
||||
("https://127.0.0.1/?param=value", ('https', '127.0.0.1', 443, '/?param=value')),
|
||||
("https://127.0.0.1:12345/", ('https', '127.0.0.1', 12345, '/')),
|
||||
|
||||
("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 80, '/foo')),
|
||||
("http://egg:7890 ", ('http', 'egg', 7890, '/')),
|
||||
)
|
||||
|
||||
for url, test in tests:
|
||||
self.assertEquals(self._parse(url), test, url)
|
||||
|
||||
def test_externalUnicodeInterference(self):
|
||||
"""
|
||||
|
|
@ -51,5 +65,228 @@ class ParseUrlTestCase(unittest.TestCase):
|
|||
self.assertTrue(isinstance(scheme, str))
|
||||
self.assertTrue(isinstance(host, str))
|
||||
self.assertTrue(isinstance(path, str))
|
||||
self.assertTrue(isinstance(port, int))
|
||||
|
||||
|
||||
|
||||
class ScrapyHTTPPageGetterTests(unittest.TestCase):
|
||||
|
||||
def test_earlyHeaders(self):
|
||||
# basic test stolen from twisted HTTPageGetter
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
url='http://foo/bar',
|
||||
body="some data",
|
||||
headers={
|
||||
'Host': 'example.net',
|
||||
'User-Agent': 'fooble',
|
||||
'Cookie': 'blah blah',
|
||||
'Content-Length': '12981',
|
||||
'Useful': 'value'})
|
||||
|
||||
self._test(factory,
|
||||
"GET /bar HTTP/1.0\r\n"
|
||||
"Content-Length: 9\r\n"
|
||||
"Useful: value\r\n"
|
||||
"Connection: close\r\n"
|
||||
"User-Agent: fooble\r\n"
|
||||
"Host: example.net\r\n"
|
||||
"Cookie: blah blah\r\n"
|
||||
"\r\n"
|
||||
"some data")
|
||||
|
||||
# test minimal sent headers
|
||||
factory = client.ScrapyHTTPClientFactory('http://foo/bar')
|
||||
self._test(factory,
|
||||
"GET /bar HTTP/1.0\r\n"
|
||||
"Host: foo\r\n"
|
||||
"\r\n")
|
||||
|
||||
# test a simple POST with body and content-type
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
method='POST',
|
||||
url='http://foo/bar',
|
||||
body='name=value',
|
||||
headers={'Content-Type': 'application/x-www-form-urlencoded'})
|
||||
|
||||
self._test(factory,
|
||||
"POST /bar HTTP/1.0\r\n"
|
||||
"Host: foo\r\n"
|
||||
"Connection: close\r\n"
|
||||
"Content-Type: application/x-www-form-urlencoded\r\n"
|
||||
"Content-Length: 10\r\n"
|
||||
"\r\n"
|
||||
"name=value")
|
||||
|
||||
# test with single and multivalued headers
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
url='http://foo/bar',
|
||||
headers={
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
})
|
||||
|
||||
self._test(factory,
|
||||
"GET /bar HTTP/1.0\r\n"
|
||||
"Host: foo\r\n"
|
||||
"X-Meta-Multivalued: value1\r\n"
|
||||
"X-Meta-Multivalued: value2\r\n"
|
||||
"X-Meta-Single: single\r\n"
|
||||
"\r\n")
|
||||
|
||||
# same test with single and multivalued headers but using Headers class
|
||||
factory = client.ScrapyHTTPClientFactory(
|
||||
url='http://foo/bar',
|
||||
headers=Headers({
|
||||
'X-Meta-Single': 'single',
|
||||
'X-Meta-Multivalued': ['value1', 'value2'],
|
||||
}))
|
||||
|
||||
self._test(factory,
|
||||
"GET /bar HTTP/1.0\r\n"
|
||||
"Host: foo\r\n"
|
||||
"X-Meta-Multivalued: value1\r\n"
|
||||
"X-Meta-Multivalued: value2\r\n"
|
||||
"X-Meta-Single: single\r\n"
|
||||
"\r\n")
|
||||
|
||||
def _test(self, factory, testvalue):
|
||||
transport = StringTransport()
|
||||
protocol = client.ScrapyHTTPPageGetter()
|
||||
protocol.factory = factory
|
||||
protocol.makeConnection(transport)
|
||||
self.assertEqual(transport.value(), testvalue)
|
||||
return testvalue
|
||||
|
||||
|
||||
from twisted.web.test.test_webclient import ForeverTakingResource, \
|
||||
ErrorResource, NoLengthResource, HostHeaderResource, \
|
||||
PayloadResource, BrokenDownloadResource
|
||||
|
||||
class WebClientTestCase(unittest.TestCase):
|
||||
def _listen(self, site):
|
||||
return reactor.listenTCP(0, site, interface="127.0.0.1")
|
||||
|
||||
def setUp(self):
|
||||
name = self.mktemp()
|
||||
os.mkdir(name)
|
||||
FilePath(name).child("file").setContent("0123456789")
|
||||
r = static.File(name)
|
||||
r.putChild("redirect", util.Redirect("/file"))
|
||||
r.putChild("wait", ForeverTakingResource())
|
||||
r.putChild("error", ErrorResource())
|
||||
r.putChild("nolength", NoLengthResource())
|
||||
r.putChild("host", HostHeaderResource())
|
||||
r.putChild("payload", PayloadResource())
|
||||
r.putChild("broken", BrokenDownloadResource())
|
||||
self.site = server.Site(r, timeout=None)
|
||||
self.wrapper = WrappingFactory(self.site)
|
||||
self.port = self._listen(self.wrapper)
|
||||
self.portno = self.port.getHost().port
|
||||
|
||||
def tearDown(self):
|
||||
return self.port.stopListening()
|
||||
|
||||
def getURL(self, path):
|
||||
return "http://127.0.0.1:%d/%s" % (self.portno, path)
|
||||
|
||||
def testPayload(self):
|
||||
s = "0123456789" * 10
|
||||
return client.getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s)
|
||||
|
||||
def testBrokenDownload(self):
|
||||
# test what happens when download gets disconnected in the middle
|
||||
d = client.getPage(self.getURL("broken"))
|
||||
d = self.assertFailure(d, client.PartialDownloadError)
|
||||
d.addCallback(lambda exc: self.assertEquals(exc.response, "abc"))
|
||||
return d
|
||||
|
||||
def testHostHeader(self):
|
||||
# if we pass Host header explicitly, it should be used, otherwise
|
||||
# it should extract from url
|
||||
return defer.gatherResults([
|
||||
client.getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1"),
|
||||
client.getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")])
|
||||
|
||||
|
||||
def test_getPage(self):
|
||||
"""
|
||||
L{client.getPage} returns a L{Deferred} which is called back with
|
||||
the body of the response if the default method B{GET} is used.
|
||||
"""
|
||||
d = client.getPage(self.getURL("file"))
|
||||
d.addCallback(self.assertEquals, "0123456789")
|
||||
return d
|
||||
|
||||
|
||||
def test_getPageHead(self):
|
||||
"""
|
||||
L{client.getPage} returns a L{Deferred} which is called back with
|
||||
the empty string if the method is C{HEAD} and there is a successful
|
||||
response code.
|
||||
"""
|
||||
def getPage(method):
|
||||
return client.getPage(self.getURL("file"), method=method)
|
||||
return defer.gatherResults([
|
||||
getPage("head").addCallback(self.assertEqual, ""),
|
||||
getPage("HEAD").addCallback(self.assertEqual, "")])
|
||||
|
||||
|
||||
def test_timeoutNotTriggering(self):
|
||||
"""
|
||||
When a non-zero timeout is passed to L{getPage} and the page is
|
||||
retrieved before the timeout period elapses, the L{Deferred} is
|
||||
called back with the contents of the page.
|
||||
"""
|
||||
d = client.getPage(self.getURL("host"), timeout=100)
|
||||
d.addCallback(self.assertEquals, "127.0.0.1")
|
||||
return d
|
||||
|
||||
|
||||
def test_timeoutTriggering(self):
|
||||
"""
|
||||
When a non-zero timeout is passed to L{getPage} and that many
|
||||
seconds elapse before the server responds to the request. the
|
||||
L{Deferred} is errbacked with a L{error.TimeoutError}.
|
||||
"""
|
||||
finished = self.assertFailure(
|
||||
client.getPage(self.getURL("wait"), timeout=0.000001),
|
||||
defer.TimeoutError)
|
||||
def cleanup(passthrough):
|
||||
# Clean up the server which is hanging around not doing
|
||||
# anything.
|
||||
connected = self.wrapper.protocols.keys()
|
||||
# There might be nothing here if the server managed to already see
|
||||
# that the connection was lost.
|
||||
if connected:
|
||||
connected[0].transport.loseConnection()
|
||||
return passthrough
|
||||
finished.addBoth(cleanup)
|
||||
return finished
|
||||
|
||||
def testNotFound(self):
|
||||
return client.getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile)
|
||||
|
||||
def _cbNoSuchFile(self, pageData):
|
||||
self.assert_('404 - No Such Resource' in pageData)
|
||||
|
||||
def testFactoryInfo(self):
|
||||
url = self.getURL('file')
|
||||
scheme, host, port, path = client._parse(url)
|
||||
factory = client.HTTPClientFactory(url)
|
||||
reactor.connectTCP(host, port, factory)
|
||||
return factory.deferred.addCallback(self._cbFactoryInfo, factory)
|
||||
|
||||
def _cbFactoryInfo(self, ignoredResult, factory):
|
||||
self.assertEquals(factory.status, '200')
|
||||
self.assert_(factory.version.startswith('HTTP/'))
|
||||
self.assertEquals(factory.message, 'OK')
|
||||
self.assertEquals(factory.response_headers['content-length'][0], '10')
|
||||
|
||||
def testRedirect(self):
|
||||
return client.getPage(self.getURL("redirect")).addCallback(self._cbRedirect)
|
||||
|
||||
def _cbRedirect(self, pageData):
|
||||
self.assertEquals(pageData,
|
||||
'\n<html>\n <head>\n <meta http-equiv="refresh" content="0;URL=/file">\n'
|
||||
' </head>\n <body bgcolor="#FFFFFF" text="#000000">\n '
|
||||
'<a href="/file">click here</a>\n </body>\n</html>\n')
|
||||
|
|
|
|||
Loading…
Reference in New Issue