From 8aa7d153ae0938539654bdfba3b4edcc23c34d81 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Mon, 5 Oct 2009 04:10:22 -0200 Subject: [PATCH] rewrote of downloader handlers * add REQUEST_HANDLERS setting with defaults for file, http and https schemes * add documentation of new setting * add unittests for all the builtin handlers * remove unused getPage function --- docs/topics/settings.rst | 27 +++++ scrapy/conf/default_settings.py | 7 ++ scrapy/core/downloader/handlers.py | 90 ----------------- scrapy/core/downloader/handlers/__init__.py | 27 +++++ scrapy/core/downloader/handlers/file.py | 18 ++++ scrapy/core/downloader/handlers/http.py | 50 ++++++++++ scrapy/core/downloader/webclient.py | 69 +++++-------- scrapy/tests/test_downloader_handlers.py | 105 ++++++++++++++++++++ scrapy/tests/test_webclient.py | 63 +++++++----- 9 files changed, 299 insertions(+), 157 deletions(-) delete mode 100644 scrapy/core/downloader/handlers.py create mode 100644 scrapy/core/downloader/handlers/__init__.py create mode 100644 scrapy/core/downloader/handlers/file.py create mode 100644 scrapy/core/downloader/handlers/http.py create mode 100644 scrapy/tests/test_downloader_handlers.py diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 79b679fad..5cd6ff5b2 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -725,6 +725,33 @@ Default: ``+2`` Adjust redirect request priority relative to original request. A negative priority adjust means more priority. +.. setting:: REQUEST_HANDLERS + +REQUEST_HANDLERS +---------------- + +Default: ``{}`` + +A dict containing the request downloader handlers enabled in your project. +See `REQUEST_HANDLERS_BASE` for example format. + +.. setting:: REQUEST_HANDLERS_BASE + +REQUEST_HANDLERS_BASE +--------------------- + +Default:: + + { + 'file': 'scrapy.core.downloader.handlers.file.download_file', + 'http': 'scrapy.core.downloader.handlers.http.download_http', + 'https': 'scrapy.core.downloader.handlers.http.download_http', + } + +A dict containing the request download handlers enabled by default in Scrapy. +You should never modify this setting in your project, modify +:setting:`REQUEST_HANDLERS` instead. + .. setting:: REQUESTS_PER_DOMAIN REQUESTS_PER_DOMAIN diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index 17b4f7db0..e31ca2783 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -124,6 +124,13 @@ REDIRECT_MAX_METAREFRESH_DELAY = 100 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 +REQUEST_HANDLERS = {} +REQUEST_HANDLERS_BASE = { + 'file': 'scrapy.core.downloader.handlers.file.download_file', + 'http': 'scrapy.core.downloader.handlers.http.download_http', + 'https': 'scrapy.core.downloader.handlers.http.download_http', +} + REQUESTS_QUEUE_SIZE = 0 REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain diff --git a/scrapy/core/downloader/handlers.py b/scrapy/core/downloader/handlers.py deleted file mode 100644 index 016e6677d..000000000 --- a/scrapy/core/downloader/handlers.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -Download handlers for different schemes -""" -from __future__ import with_statement - -import urlparse - -from twisted.internet import reactor -try: - from twisted.internet import ssl -except ImportError: - pass - -from scrapy import optional_features -from scrapy.core import signals -from scrapy.http import Headers -from scrapy.core.exceptions import NotSupported -from scrapy.utils.defer import defer_succeed -from scrapy.utils.httpobj import urlparse_cached -from scrapy.utils.signal import send_catch_log -from scrapy.utils.misc import load_object -from scrapy.core.downloader.responsetypes import responsetypes -from scrapy.conf import settings - - -HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) -default_timeout = settings.getint('DOWNLOAD_TIMEOUT') -ssl_supported = 'ssl' in optional_features - -def download_any(request, spider): - scheme = urlparse_cached(request).scheme - if scheme == 'http': - return download_http(request, spider) - elif scheme == 'https': - if ssl_supported: - return download_https(request, spider) - else: - raise NotSupported("HTTPS not supported: install pyopenssl library") - elif scheme == 'file': - return download_file(request, spider) - else: - raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url)) - -def create_factory(request, spider): - """Return HTTPClientFactory for the given Request""" - url = urlparse.urldefrag(request.url)[0] - timeout = getattr(spider, "download_timeout", None) or default_timeout - factory = HTTPClientFactory.from_request(request, timeout) - - def _create_response(body): - body = body or '' - status = int(factory.status) - headers = Headers(factory.response_headers) - respcls = responsetypes.from_args(headers=headers, url=url) - r = respcls(url=request.url, status=status, headers=headers, body=body) - send_catch_log(signal=signals.request_uploaded, sender='download_http', \ - request=request, spider=spider) - send_catch_log(signal=signals.response_downloaded, sender='download_http', \ - response=r, spider=spider) - return r - - factory.deferred.addCallbacks(_create_response) - return factory - -def download_http(request, spider): - """Return a deferred for the HTTP download""" - factory = create_factory(request, spider) - url = urlparse_cached(request) - port = url.port - reactor.connectTCP(url.hostname, port or 80, factory) - return factory.deferred - -def download_https(request, spider): - """Return a deferred for the HTTPS download""" - factory = create_factory(request, spider) - url = urlparse_cached(request) - port = url.port - contextFactory = ssl.ClientContextFactory() - reactor.connectSSL(url.hostname, port or 443, factory, contextFactory) - return factory.deferred - -def download_file(request, spider) : - """Return a deferred for a file download.""" - filepath = request.url.split("file://")[1] - with open(filepath) as f: - body = f.read() - respcls = responsetypes.from_args(filename=filepath, body=body) - response = respcls(url=request.url, body=body) - - return defer_succeed(response) diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py new file mode 100644 index 000000000..94d619202 --- /dev/null +++ b/scrapy/core/downloader/handlers/__init__.py @@ -0,0 +1,27 @@ +"""Download handlers for different schemes""" + +from scrapy.core.exceptions import NotSupported +from scrapy.utils.httpobj import urlparse_cached +from scrapy.conf import settings +from scrapy.utils.misc import load_object + + +class RequestHandlers(object): + + def __init__(self): + self._handlers = {} + handlers = settings.get('REQUEST_HANDLERS_BASE') + handlers.update(settings.get('REQUEST_HANDLERS', {})) + for scheme, cls in handlers.iteritems(): + self._handlers[scheme] = load_object(cls) + + def download_request(self, request, spider): + scheme = urlparse_cached(request).scheme + try: + handler = self._handlers[scheme] + except KeyError: + raise NotSupported("Unsupported URL scheme '%s' in: <%s>" % (scheme, request.url)) + return handler(request, spider) + + +download_any = RequestHandlers().download_request diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py new file mode 100644 index 000000000..8a02d02d6 --- /dev/null +++ b/scrapy/core/downloader/handlers/file.py @@ -0,0 +1,18 @@ +"""Download handler for file:// scheme""" +from __future__ import with_statement + +from twisted.internet import defer +from scrapy.core.downloader.responsetypes import responsetypes + + +def download_file(request, spider): + """Return a deferred for a file download.""" + return defer.maybeDeferred(_all_in_one_read_download_file, request, spider) + +def _all_in_one_read_download_file(request, spider): + filepath = request.url.split("file://")[1] + with open(filepath) as f: + body = f.read() + respcls = responsetypes.from_args(filename=filepath, body=body) + return respcls(url=request.url, body=body) + diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py new file mode 100644 index 000000000..756bde6e3 --- /dev/null +++ b/scrapy/core/downloader/handlers/http.py @@ -0,0 +1,50 @@ +"""Download handlers for http and https schemes""" + +from twisted.internet import reactor + +from scrapy.core import signals +from scrapy.core.exceptions import NotSupported +from scrapy.utils.signal import send_catch_log +from scrapy.utils.misc import load_object +from scrapy.conf import settings +from scrapy import optional_features + +ssl_supported = 'ssl' in optional_features +if ssl_supported: + from twisted.internet.ssl import ClientContextFactory + + +HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) +default_timeout = settings.getint('DOWNLOAD_TIMEOUT') + +def _create_factory(request, spider): + def _download_signals(response): + send_catch_log(signal=signals.request_uploaded, \ + sender='download_http', request=request, spider=spider) + send_catch_log(signal=signals.response_downloaded, \ + sender='download_http', response=response, spider=spider) + return response + + timeout = getattr(spider, "download_timeout", None) or default_timeout + factory = HTTPClientFactory(request, timeout) + factory.deferred.addCallbacks(_download_signals) + return factory + + +def _connect(factory): + host, port = factory.host, factory.port + if factory.scheme == 'https': + if ssl_supported: + return reactor.connectSSL(host, port, factory, ClientContextFactory()) + raise NotSupported("HTTPS not supported: install pyopenssl library") + else: + return reactor.connectTCP(host, port, factory) + + +def download_http(request, spider): + """Return a deferred for the HTTP download""" + factory = _create_factory(request, spider) + _connect(factory) + return factory.deferred + + diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index ead74ae79..6eaffb5b0 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,12 +1,14 @@ -from urlparse import urlparse, urlunparse +from urlparse import urlparse, urlunparse, urldefrag from twisted.python import failure -from twisted.web.client import HTTPClientFactory, PartialDownloadError +from twisted.web.client import PartialDownloadError, HTTPClientFactory from twisted.web.http import HTTPClient from twisted.internet import defer from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached +from scrapy.core.downloader.responsetypes import responsetypes + def _parsed_url_args(parsed): path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) @@ -18,11 +20,13 @@ def _parsed_url_args(parsed): port = 443 if scheme == 'https' else 80 return scheme, netloc, host, port, path + def _parse(url): url = url.strip() parsed = urlparse(url) return _parsed_url_args(parsed) + class ScrapyHTTPPageGetter(HTTPClient): def connectionMade(self): @@ -64,7 +68,9 @@ class ScrapyHTTPPageGetter(HTTPClient): def timeout(self): self.transport.loseConnection() - self.factory.noPage(defer.TimeoutError("Getting %s took longer than %s seconds." % (self.factory.url, self.factory.timeout))) + self.factory.noPage(\ + defer.TimeoutError("Getting %s took longer than %s seconds." % \ + (self.factory.url, self.factory.timeout))) class ScrapyHTTPClientFactory(HTTPClientFactory): @@ -74,22 +80,19 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): """ protocol = ScrapyHTTPPageGetter - response_headers = None waiting = 1 noisy = False - def __init__(self, url, method='GET', body=None, headers=None, timeout=0, parsedurl=None): - self.url = url - self.method = method - self.body = body or None - if parsedurl: - self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsedurl) - else: - self.scheme, self.netloc, self.host, self.port, self.path = _parse(url) - + def __init__(self, request, timeout=0): + self.url = urldefrag(request.url)[0] + self.method = request.method + self.body = request.body or None + self.headers = request.headers + self.response_headers = None self.timeout = timeout - self.headers = Headers(headers or {}) - self.deferred = defer.Deferred() + self.deferred = defer.Deferred().addCallback(self._build_response) + + self._set_connection_attributes(request) # set Host header based on url self.headers.setdefault('Host', self.netloc) @@ -100,33 +103,15 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): # just in case a broken http/1.1 decides to keep connection alive self.headers.setdefault("Connection", "close") - @classmethod - def from_request(cls, request, timeout): - return cls(request.url, - method=request.method, - body=request.body or None, # see http://dev.scrapy.org/ticket/60 - headers=Headers(request.headers or {}), - timeout=timeout, - parsedurl=urlparse_cached(request), - ) + def _build_response(self, body): + status = int(self.status) + headers = Headers(self.response_headers) + respcls = responsetypes.from_args(headers=headers, url=self.url) + return respcls(url=self.url, status=status, headers=headers, body=body) + + def _set_connection_attributes(self, request): + parsed = urlparse_cached(request) + self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed) def gotHeaders(self, headers): self.response_headers = headers - - - -def getPage(url, contextFactory=None, *args, **kwargs): - """ - Download a web page as a string. - - Download a page. Return a deferred, which will callback with a - page (as a string) or errback with a description of the error. - - See HTTPClientFactory to see what extra args can be passed. - """ - from twisted.web.client import _makeGetterFactory - return _makeGetterFactory( - url, - ScrapyHTTPClientFactory, - contextFactory=contextFactory, - *args, **kwargs).deferred diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py new file mode 100644 index 000000000..11fc3bbdf --- /dev/null +++ b/scrapy/tests/test_downloader_handlers.py @@ -0,0 +1,105 @@ +import os + +from twisted.trial import unittest +from twisted.protocols.policies import WrappingFactory +from twisted.python.filepath import FilePath +from twisted.internet import reactor, defer +from twisted.web import server, static, util, resource +from twisted.web.test.test_webclient import ForeverTakingResource, \ + NoLengthResource, HostHeaderResource, \ + PayloadResource, BrokenDownloadResource + +from scrapy.core.downloader.webclient import PartialDownloadError +from scrapy.core.downloader.handlers.file import download_file +from scrapy.core.downloader.handlers.http import download_http +from scrapy.spider import BaseSpider +from scrapy.http import Request + + +class FileTestCase(unittest.TestCase): + + def setUp(self): + self.tmpname = self.mktemp() + fd = open(self.tmpname, 'w') + fd.write('0123456789') + fd.close() + + def test_download(self): + def _test(response): + self.assertEquals(response.url, request.url) + self.assertEquals(response.status, 200) + self.assertEquals(response.body, '0123456789') + + request = Request('file://%s' % self.tmpname) + return download_file(request, BaseSpider()).addCallback(_test) + + def test_non_existent(self): + request = Request('file://%s' % self.mktemp()) + d = download_file(request, BaseSpider()) + return self.assertFailure(d, IOError) + + +class HttpTestCase(unittest.TestCase): + + def setUp(self): + name = self.mktemp() + os.mkdir(name) + FilePath(name).child("file").setContent("0123456789") + r = static.File(name) + r.putChild("redirect", util.Redirect("/file")) + r.putChild("wait", ForeverTakingResource()) + r.putChild("nolength", NoLengthResource()) + r.putChild("host", HostHeaderResource()) + r.putChild("payload", PayloadResource()) + r.putChild("broken", BrokenDownloadResource()) + self.site = server.Site(r, timeout=None) + self.wrapper = WrappingFactory(self.site) + self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1') + self.portno = self.port.getHost().port + + def tearDown(self): + return self.port.stopListening() + + def getURL(self, path): + return "http://127.0.0.1:%d/%s" % (self.portno, path) + + def test_download(self): + request = Request(self.getURL('file')) + d = download_http(request, BaseSpider()) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEquals, "0123456789") + return d + + def test_redirect_status(self): + request = Request(self.getURL('redirect')) + d = download_http(request, BaseSpider()) + d.addCallback(lambda r: r.status) + d.addCallback(self.assertEquals, 302) + return d + + def test_timeout_download_from_spider(self): + spider = BaseSpider() + spider.download_timeout = 0.000001 + request = Request(self.getURL('wait')) + d = download_http(request, spider) + return self.assertFailure(d, defer.TimeoutError) + + def test_host_header(self): + request = Request(self.getURL('host')) + d = download_http(request, BaseSpider()) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEquals, '127.0.0.1:%d' % self.portno) + return d + + def test_payload(self): + body = '1'*100 # PayloadResource requires body length to be 100 + request = Request(self.getURL('payload'), method='POST', body=body) + d = download_http(request, BaseSpider()) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEquals, body) + return d + + def test_broken_download(self): + request = Request(self.getURL('broken')) + d = download_http(request, BaseSpider()) + return self.assertFailure(d, PartialDownloadError) diff --git a/scrapy/tests/test_webclient.py b/scrapy/tests/test_webclient.py index fa0799657..260b2bc08 100644 --- a/scrapy/tests/test_webclient.py +++ b/scrapy/tests/test_webclient.py @@ -13,14 +13,27 @@ from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from scrapy.core.downloader import webclient as client -from scrapy.http import Headers +from scrapy.http import Request, Headers + + +def getPage(url, contextFactory=None, *args, **kwargs): + """Adapted version of twisted.web.client.getPage""" + def _clientfactory(*args, **kwargs): + timeout = kwargs.pop('timeout', 0) + f = client.ScrapyHTTPClientFactory(Request(*args, **kwargs), timeout=timeout) + f.deferred.addCallback(lambda r: r.body) + return f + + from twisted.web.client import _makeGetterFactory + return _makeGetterFactory(url, _clientfactory, + contextFactory=contextFactory, *args, **kwargs).deferred class ParseUrlTestCase(unittest.TestCase): """Test URL parsing facility and defaults values.""" def _parse(self, url): - f = client.ScrapyHTTPClientFactory(url) + f = client.ScrapyHTTPClientFactory(Request(url)) return (f.scheme, f.netloc, f.host, f.port, f.path) def testParse(self): @@ -75,7 +88,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): def test_earlyHeaders(self): # basic test stolen from twisted HTTPageGetter - factory = client.ScrapyHTTPClientFactory( + factory = client.ScrapyHTTPClientFactory(Request( url='http://foo/bar', body="some data", headers={ @@ -83,7 +96,7 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): 'User-Agent': 'fooble', 'Cookie': 'blah blah', 'Content-Length': '12981', - 'Useful': 'value'}) + 'Useful': 'value'})) self._test(factory, "GET /bar HTTP/1.0\r\n" @@ -97,18 +110,18 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): "some data") # test minimal sent headers - factory = client.ScrapyHTTPClientFactory('http://foo/bar') + factory = client.ScrapyHTTPClientFactory(Request('http://foo/bar')) self._test(factory, "GET /bar HTTP/1.0\r\n" "Host: foo\r\n" "\r\n") # test a simple POST with body and content-type - factory = client.ScrapyHTTPClientFactory( + factory = client.ScrapyHTTPClientFactory(Request( method='POST', url='http://foo/bar', body='name=value', - headers={'Content-Type': 'application/x-www-form-urlencoded'}) + headers={'Content-Type': 'application/x-www-form-urlencoded'})) self._test(factory, "POST /bar HTTP/1.0\r\n" @@ -120,12 +133,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): "name=value") # test with single and multivalued headers - factory = client.ScrapyHTTPClientFactory( + factory = client.ScrapyHTTPClientFactory(Request( url='http://foo/bar', headers={ 'X-Meta-Single': 'single', 'X-Meta-Multivalued': ['value1', 'value2'], - }) + })) self._test(factory, "GET /bar HTTP/1.0\r\n" @@ -136,12 +149,12 @@ class ScrapyHTTPPageGetterTests(unittest.TestCase): "\r\n") # same test with single and multivalued headers but using Headers class - factory = client.ScrapyHTTPClientFactory( + factory = client.ScrapyHTTPClientFactory(Request( url='http://foo/bar', headers=Headers({ 'X-Meta-Single': 'single', 'X-Meta-Multivalued': ['value1', 'value2'], - })) + }))) self._test(factory, "GET /bar HTTP/1.0\r\n" @@ -193,11 +206,11 @@ class WebClientTestCase(unittest.TestCase): def testPayload(self): s = "0123456789" * 10 - return client.getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s) + return getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s) def testBrokenDownload(self): # test what happens when download gets disconnected in the middle - d = client.getPage(self.getURL("broken")) + d = getPage(self.getURL("broken")) d = self.assertFailure(d, client.PartialDownloadError) d.addCallback(lambda exc: self.assertEquals(exc.response, "abc")) return d @@ -206,8 +219,8 @@ class WebClientTestCase(unittest.TestCase): # if we pass Host header explicitly, it should be used, otherwise # it should extract from url return defer.gatherResults([ - client.getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno), - client.getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")]) + getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno), + getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")]) def test_getPage(self): @@ -215,7 +228,7 @@ class WebClientTestCase(unittest.TestCase): L{client.getPage} returns a L{Deferred} which is called back with the body of the response if the default method B{GET} is used. """ - d = client.getPage(self.getURL("file")) + d = getPage(self.getURL("file")) d.addCallback(self.assertEquals, "0123456789") return d @@ -226,11 +239,11 @@ class WebClientTestCase(unittest.TestCase): the empty string if the method is C{HEAD} and there is a successful response code. """ - def getPage(method): - return client.getPage(self.getURL("file"), method=method) + def _getPage(method): + return getPage(self.getURL("file"), method=method) return defer.gatherResults([ - getPage("head").addCallback(self.assertEqual, ""), - getPage("HEAD").addCallback(self.assertEqual, "")]) + _getPage("head").addCallback(self.assertEqual, ""), + _getPage("HEAD").addCallback(self.assertEqual, "")]) def test_timeoutNotTriggering(self): @@ -239,7 +252,7 @@ class WebClientTestCase(unittest.TestCase): retrieved before the timeout period elapses, the L{Deferred} is called back with the contents of the page. """ - d = client.getPage(self.getURL("host"), timeout=100) + d = getPage(self.getURL("host"), timeout=100) d.addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno) return d @@ -251,7 +264,7 @@ class WebClientTestCase(unittest.TestCase): L{Deferred} is errbacked with a L{error.TimeoutError}. """ finished = self.assertFailure( - client.getPage(self.getURL("wait"), timeout=0.000001), + getPage(self.getURL("wait"), timeout=0.000001), defer.TimeoutError) def cleanup(passthrough): # Clean up the server which is hanging around not doing @@ -266,7 +279,7 @@ class WebClientTestCase(unittest.TestCase): return finished def testNotFound(self): - return client.getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile) + return getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile) def _cbNoSuchFile(self, pageData): self.assert_('404 - No Such Resource' in pageData) @@ -274,7 +287,7 @@ class WebClientTestCase(unittest.TestCase): def testFactoryInfo(self): url = self.getURL('file') scheme, netloc, host, port, path = client._parse(url) - factory = client.ScrapyHTTPClientFactory(url) + factory = client.ScrapyHTTPClientFactory(Request(url)) reactor.connectTCP(host, port, factory) return factory.deferred.addCallback(self._cbFactoryInfo, factory) @@ -285,7 +298,7 @@ class WebClientTestCase(unittest.TestCase): self.assertEquals(factory.response_headers['content-length'], '10') def testRedirect(self): - return client.getPage(self.getURL("redirect")).addCallback(self._cbRedirect) + return getPage(self.getURL("redirect")).addCallback(self._cbRedirect) def _cbRedirect(self, pageData): self.assertEquals(pageData,