diff --git a/scrapy/trunk/scrapy/core/downloader/handlers.py b/scrapy/trunk/scrapy/core/downloader/handlers.py index 21120fc5d..51b94563f 100644 --- a/scrapy/trunk/scrapy/core/downloader/handlers.py +++ b/scrapy/trunk/scrapy/core/downloader/handlers.py @@ -25,7 +25,6 @@ from scrapy.core.downloader.responsetypes import responsetypes from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory as HTTPClientFactory default_timeout = settings.getint('DOWNLOAD_TIMEOUT') -default_agent = settings.get('USER_AGENT') ssl_supported = 'ssl' in optional_features # Cache for dns lookups. @@ -48,15 +47,13 @@ def download_any(request, spider): def create_factory(request, spider): """Return HTTPClientFactory for the given Request""" url = urlparse.urldefrag(request.url)[0] + timeout = getattr(spider, "download_timeout", None) or default_timeout - agent = request.headers.pop('user-agent', default_agent) factory = HTTPClientFactory(url=url, # never pass unicode urls to twisted method=request.method, - postdata=request.body or None, # see http://dev.scrapy.org/ticket/60 + body=request.body or None, # see http://dev.scrapy.org/ticket/60 headers=request.headers, - agent=agent, - timeout=getattr(spider, "download_timeout", None) or default_timeout, - followRedirect=False) + timeout=timeout) def _create_response(body): body = body or '' @@ -69,16 +66,12 @@ def create_factory(request, spider): return r def _on_success(body): - return _create_response(body) + response = _create_response(body) + if response.status not in (200, 201, 202): + raise HttpException(response.status, None, response) + return response - def _on_error(_failure): - ex = _failure.value - if isinstance(ex, web_error.Error): # HttpException - raise HttpException(ex.status, ex.message, _create_response(ex.response)) - return _failure - - factory.noisy = False - factory.deferred.addCallbacks(_on_success, _on_error) + factory.deferred.addCallbacks(_on_success) return factory def download_http(request, spider): diff --git a/scrapy/trunk/scrapy/core/downloader/webclient.py b/scrapy/trunk/scrapy/core/downloader/webclient.py index 9a6080043..b1064b29e 100644 --- a/scrapy/trunk/scrapy/core/downloader/webclient.py +++ b/scrapy/trunk/scrapy/core/downloader/webclient.py @@ -1,152 +1,121 @@ from urlparse import urlunparse -from twisted.web.client import HTTPClientFactory, HTTPPageGetter -from twisted.web import http from twisted.python import failure -from twisted.web import error +from twisted.web.client import HTTPClientFactory, PartialDownloadError +from twisted.web.http import HTTPClient from twisted.internet import defer from scrapy.http import Url, Headers -from scrapy.utils.misc import arg_to_iter -def _parse(url, defaultPort=None): + +def _parse(url): url = url.strip() try: parsed = url.parsedurl except AttributeError: - parsed = Url(url).parsedurl + parsed = Url(url.strip()).parsedurl + + path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) + host = parsed.hostname + port = parsed.port + scheme = parsed.scheme + + if port is None: + port = 443 if scheme == 'https' else 80 - scheme = parsed[0] - path = urlunparse(('','')+parsed[2:]) - if defaultPort is None: - if scheme == 'https': - defaultPort = 443 - else: - defaultPort = 80 - host, port = parsed[1], defaultPort - if ':' in host: - host, port = host.split(':') - port = int(port) - if path == "": - path = "/" return scheme, host, port, path -class ScrapyHTTPPageGetter(HTTPPageGetter): - quietLoss = 0 - failed = 0 - - _specialHeaders = set(('host', 'user-agent', 'content-length')) +class ScrapyHTTPPageGetter(HTTPClient): def connectionMade(self): - headers = self.factory.headers - method = getattr(self.factory, 'method', 'GET') + self.headers = Headers() # bucket for response headers - self.sendCommand(method, self.factory.path) - self.sendHeader('Host', headers.get("host", self.factory.host)) - self.sendHeader('User-Agent', headers.get('User-Agent', self.factory.agent)) - - data = getattr(self.factory, 'postdata', None) - if data is not None: - self.sendHeader("Content-Length", str(len(data))) - - for key, value in self.factory.headers.items(): - if key.lower() not in self._specialHeaders: + # Method command + self.sendCommand(self.factory.method, self.factory.path) + # Headers + for key, values in self.factory.headers.items(): + for value in values: self.sendHeader(key, value) - self.endHeaders() - self.headers = Headers() - - if data is not None: - self.transport.write(data) - - def sendHeader(self, name, value): - for v in arg_to_iter(value): - self.transport.write('%s: %s\r\n' % (name, v)) + # Body + if self.factory.body is not None: + self.transport.write(self.factory.body) def handleHeader(self, key, value): self.headers.appendlist(key, value) def handleStatus(self, version, status, message): - self.version, self.status, self.message = version, status, message self.factory.gotStatus(version, status, message) def handleEndHeaders(self): self.factory.gotHeaders(self.headers) - m = getattr(self, 'handleStatus_'+self.status, self.handleStatusDefault) - m() - - def handleStatus_200(self): - pass - - handleStatus_201 = lambda self: self.handleStatus_200() - handleStatus_202 = lambda self: self.handleStatus_200() - - def handleStatusDefault(self): - self.failed = 1 def connectionLost(self, reason): - if not self.quietLoss: - http.HTTPClient.connectionLost(self, reason) - self.factory.noPage(reason) + HTTPClient.connectionLost(self, reason) + self.factory.noPage(reason) def handleResponse(self, response): - if self.quietLoss: - return - - if self.failed: - self.factory.noPage(failure.Failure(error.Error(self.status, self.message, response))) - if self.factory.method.upper() == 'HEAD': - # Callback with empty string, since there is never a response - # body for HEAD requests. self.factory.page('') elif self.length != None and self.length != 0: self.factory.noPage(failure.Failure( - PartialDownloadError(self.status, self.message, response))) + PartialDownloadError(self.factory.status, None, response))) else: self.factory.page(response) - - # server might be stupid and not close connection. admittedly - # the fact we do only one request per connection is also - # stupid... self.transport.loseConnection() def timeout(self): - self.quietLoss = True self.transport.loseConnection() self.factory.noPage(defer.TimeoutError("Getting %s took longer than %s seconds." % (self.factory.url, self.factory.timeout))) - class ScrapyHTTPClientFactory(HTTPClientFactory): """Scrapy implementation of the HTTPClientFactory overwriting the serUrl method to make use of our Url object that cache the parse - result. Also we override gotHeaders that dies when parsing malformed - cookies. + result. """ protocol = ScrapyHTTPPageGetter + response_headers = None + waiting = 1 + noisy = False - def setURL(self, url): + def __init__(self, url, method='GET', body=None, headers=None, timeout=0): self.url = url - scheme, host, port, path = _parse(url) - if scheme and host: - self.scheme = scheme - self.host = host - self.port = port - self.path = path + self.method = method + self.body = body or None + self.scheme, self.host, self.port, self.path = _parse(url) + self.timeout = timeout + self.headers = Headers(headers or {}) + self.deferred = defer.Deferred() + + # set Host header based on url + self.headers.setdefault('Host', self.host) + + # set Content-Length based len of body + if self.body is not None: + self.headers['Content-Length'] = len(self.body) + # just in case a broken http/1.1 decides to keep connection alive + self.headers.setdefault("Connection", "close") def gotHeaders(self, headers): - """ - HTTPClientFactory.gotHeaders dies when parsing malformed cookies, - and the crawler is getting malformed cookies from this site. - - Cookies format: - http://www.ietf.org/rfc/rfc2109.txt - - I have choosen not to filter based on this, so we don't filter invalid - values that could be managed correctly by twisted. - """ self.response_headers = headers + + +def getPage(url, contextFactory=None, *args, **kwargs): + """ + Download a web page as a string. + + Download a page. Return a deferred, which will callback with a + page (as a string) or errback with a description of the error. + + See HTTPClientFactory to see what extra args can be passed. + """ + from twisted.web.client import _makeGetterFactory + return _makeGetterFactory( + url, + ScrapyHTTPClientFactory, + contextFactory=contextFactory, + *args, **kwargs).deferred diff --git a/scrapy/trunk/scrapy/tests/test_webclient.py b/scrapy/trunk/scrapy/tests/test_webclient.py index 82acd083f..62d4c0991 100644 --- a/scrapy/trunk/scrapy/tests/test_webclient.py +++ b/scrapy/trunk/scrapy/tests/test_webclient.py @@ -2,41 +2,55 @@ from twisted.internet import defer Tests borrowed from the twisted.web.client tests. """ +import os from urlparse import urlparse from twisted.trial import unittest -from twisted.web import server, static +from twisted.web import server, static, error, util from twisted.internet import reactor, defer +from twisted.test.proto_helpers import StringTransport +from twisted.python.filepath import FilePath +from twisted.protocols.policies import WrappingFactory + +from scrapy.core.downloader import webclient as client +from scrapy.http import Url, Headers -from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory -from scrapy.http import Url class ParseUrlTestCase(unittest.TestCase): """Test URL parsing facility and defaults values.""" def _parse(self, url): - f = ScrapyHTTPClientFactory(Url(url)) + f = client.ScrapyHTTPClientFactory(Url(url)) return (f.scheme, f.host, f.port, f.path) def testParse(self): - scheme, host, port, path = self._parse("http://127.0.0.1/?param=value") - self.assertEquals(path, "/?param=value") - self.assertEquals(port, 80) - scheme, host, port, path = self._parse("http://127.0.0.1/") - self.assertEquals(path, "/") - self.assertEquals(port, 80) - scheme, host, port, path = self._parse("https://127.0.0.1/") - self.assertEquals(path, "/") - self.assertEquals(port, 443) - scheme, host, port, path = self._parse("http://spam:12345/") - self.assertEquals(port, 12345) - scheme, host, port, path = self._parse("http://foo ") - self.assertEquals(host, "foo") - self.assertEquals(path, "/") - scheme, host, port, path = self._parse("http://egg:7890") - self.assertEquals(port, 7890) - self.assertEquals(host, "egg") - self.assertEquals(path, "/") + tests = ( + ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', '127.0.0.1', 80, '/?c=v&c2=v2')), + ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', '127.0.0.1', 80, '/?c=v&c2=v2')), + ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', '127.0.0.1', 80, '/foo?c=v&c2=v2')), + ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', '127.0.0.1', 100, '/?c=v&c2=v2')), + ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', '127.0.0.1', 100, '/?c=v&c2=v2')), + ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', '127.0.0.1', 100, '/foo?c=v&c2=v2')), + + ("http://127.0.0.1", ('http', '127.0.0.1', 80, '/')), + ("http://127.0.0.1/", ('http', '127.0.0.1', 80, '/')), + ("http://127.0.0.1/foo", ('http', '127.0.0.1', 80, '/foo')), + ("http://127.0.0.1?param=value", ('http', '127.0.0.1', 80, '/?param=value')), + ("http://127.0.0.1/?param=value", ('http', '127.0.0.1', 80, '/?param=value')), + ("http://127.0.0.1:12345/foo", ('http', '127.0.0.1', 12345, '/foo')), + ("http://spam:12345/foo", ('http', 'spam', 12345, '/foo')), + ("http://spam.scrapytest.org/foo", ('http', 'spam.scrapytest.org', 80, '/foo')), + + ("https://127.0.0.1/foo", ('https', '127.0.0.1', 443, '/foo')), + ("https://127.0.0.1/?param=value", ('https', '127.0.0.1', 443, '/?param=value')), + ("https://127.0.0.1:12345/", ('https', '127.0.0.1', 12345, '/')), + + ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 80, '/foo')), + ("http://egg:7890 ", ('http', 'egg', 7890, '/')), + ) + + for url, test in tests: + self.assertEquals(self._parse(url), test, url) def test_externalUnicodeInterference(self): """ @@ -51,5 +65,228 @@ class ParseUrlTestCase(unittest.TestCase): self.assertTrue(isinstance(scheme, str)) self.assertTrue(isinstance(host, str)) self.assertTrue(isinstance(path, str)) + self.assertTrue(isinstance(port, int)) + +class ScrapyHTTPPageGetterTests(unittest.TestCase): + + def test_earlyHeaders(self): + # basic test stolen from twisted HTTPageGetter + factory = client.ScrapyHTTPClientFactory( + url='http://foo/bar', + body="some data", + headers={ + 'Host': 'example.net', + 'User-Agent': 'fooble', + 'Cookie': 'blah blah', + 'Content-Length': '12981', + 'Useful': 'value'}) + + self._test(factory, + "GET /bar HTTP/1.0\r\n" + "Content-Length: 9\r\n" + "Useful: value\r\n" + "Connection: close\r\n" + "User-Agent: fooble\r\n" + "Host: example.net\r\n" + "Cookie: blah blah\r\n" + "\r\n" + "some data") + + # test minimal sent headers + factory = client.ScrapyHTTPClientFactory('http://foo/bar') + self._test(factory, + "GET /bar HTTP/1.0\r\n" + "Host: foo\r\n" + "\r\n") + + # test a simple POST with body and content-type + factory = client.ScrapyHTTPClientFactory( + method='POST', + url='http://foo/bar', + body='name=value', + headers={'Content-Type': 'application/x-www-form-urlencoded'}) + + self._test(factory, + "POST /bar HTTP/1.0\r\n" + "Host: foo\r\n" + "Connection: close\r\n" + "Content-Type: application/x-www-form-urlencoded\r\n" + "Content-Length: 10\r\n" + "\r\n" + "name=value") + + # test with single and multivalued headers + factory = client.ScrapyHTTPClientFactory( + url='http://foo/bar', + headers={ + 'X-Meta-Single': 'single', + 'X-Meta-Multivalued': ['value1', 'value2'], + }) + + self._test(factory, + "GET /bar HTTP/1.0\r\n" + "Host: foo\r\n" + "X-Meta-Multivalued: value1\r\n" + "X-Meta-Multivalued: value2\r\n" + "X-Meta-Single: single\r\n" + "\r\n") + + # same test with single and multivalued headers but using Headers class + factory = client.ScrapyHTTPClientFactory( + url='http://foo/bar', + headers=Headers({ + 'X-Meta-Single': 'single', + 'X-Meta-Multivalued': ['value1', 'value2'], + })) + + self._test(factory, + "GET /bar HTTP/1.0\r\n" + "Host: foo\r\n" + "X-Meta-Multivalued: value1\r\n" + "X-Meta-Multivalued: value2\r\n" + "X-Meta-Single: single\r\n" + "\r\n") + + def _test(self, factory, testvalue): + transport = StringTransport() + protocol = client.ScrapyHTTPPageGetter() + protocol.factory = factory + protocol.makeConnection(transport) + self.assertEqual(transport.value(), testvalue) + return testvalue + + +from twisted.web.test.test_webclient import ForeverTakingResource, \ + ErrorResource, NoLengthResource, HostHeaderResource, \ + PayloadResource, BrokenDownloadResource + +class WebClientTestCase(unittest.TestCase): + def _listen(self, site): + return reactor.listenTCP(0, site, interface="127.0.0.1") + + def setUp(self): + name = self.mktemp() + os.mkdir(name) + FilePath(name).child("file").setContent("0123456789") + r = static.File(name) + r.putChild("redirect", util.Redirect("/file")) + r.putChild("wait", ForeverTakingResource()) + r.putChild("error", ErrorResource()) + r.putChild("nolength", NoLengthResource()) + r.putChild("host", HostHeaderResource()) + r.putChild("payload", PayloadResource()) + r.putChild("broken", BrokenDownloadResource()) + self.site = server.Site(r, timeout=None) + self.wrapper = WrappingFactory(self.site) + self.port = self._listen(self.wrapper) + self.portno = self.port.getHost().port + + def tearDown(self): + return self.port.stopListening() + + def getURL(self, path): + return "http://127.0.0.1:%d/%s" % (self.portno, path) + + def testPayload(self): + s = "0123456789" * 10 + return client.getPage(self.getURL("payload"), body=s).addCallback(self.assertEquals, s) + + def testBrokenDownload(self): + # test what happens when download gets disconnected in the middle + d = client.getPage(self.getURL("broken")) + d = self.assertFailure(d, client.PartialDownloadError) + d.addCallback(lambda exc: self.assertEquals(exc.response, "abc")) + return d + + def testHostHeader(self): + # if we pass Host header explicitly, it should be used, otherwise + # it should extract from url + return defer.gatherResults([ + client.getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1"), + client.getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")]) + + + def test_getPage(self): + """ + L{client.getPage} returns a L{Deferred} which is called back with + the body of the response if the default method B{GET} is used. + """ + d = client.getPage(self.getURL("file")) + d.addCallback(self.assertEquals, "0123456789") + return d + + + def test_getPageHead(self): + """ + L{client.getPage} returns a L{Deferred} which is called back with + the empty string if the method is C{HEAD} and there is a successful + response code. + """ + def getPage(method): + return client.getPage(self.getURL("file"), method=method) + return defer.gatherResults([ + getPage("head").addCallback(self.assertEqual, ""), + getPage("HEAD").addCallback(self.assertEqual, "")]) + + + def test_timeoutNotTriggering(self): + """ + When a non-zero timeout is passed to L{getPage} and the page is + retrieved before the timeout period elapses, the L{Deferred} is + called back with the contents of the page. + """ + d = client.getPage(self.getURL("host"), timeout=100) + d.addCallback(self.assertEquals, "127.0.0.1") + return d + + + def test_timeoutTriggering(self): + """ + When a non-zero timeout is passed to L{getPage} and that many + seconds elapse before the server responds to the request. the + L{Deferred} is errbacked with a L{error.TimeoutError}. + """ + finished = self.assertFailure( + client.getPage(self.getURL("wait"), timeout=0.000001), + defer.TimeoutError) + def cleanup(passthrough): + # Clean up the server which is hanging around not doing + # anything. + connected = self.wrapper.protocols.keys() + # There might be nothing here if the server managed to already see + # that the connection was lost. + if connected: + connected[0].transport.loseConnection() + return passthrough + finished.addBoth(cleanup) + return finished + + def testNotFound(self): + return client.getPage(self.getURL('notsuchfile')).addCallback(self._cbNoSuchFile) + + def _cbNoSuchFile(self, pageData): + self.assert_('404 - No Such Resource' in pageData) + + def testFactoryInfo(self): + url = self.getURL('file') + scheme, host, port, path = client._parse(url) + factory = client.HTTPClientFactory(url) + reactor.connectTCP(host, port, factory) + return factory.deferred.addCallback(self._cbFactoryInfo, factory) + + def _cbFactoryInfo(self, ignoredResult, factory): + self.assertEquals(factory.status, '200') + self.assert_(factory.version.startswith('HTTP/')) + self.assertEquals(factory.message, 'OK') + self.assertEquals(factory.response_headers['content-length'][0], '10') + + def testRedirect(self): + return client.getPage(self.getURL("redirect")).addCallback(self._cbRedirect) + + def _cbRedirect(self, pageData): + self.assertEquals(pageData, + '\n\n \n \n' + ' \n \n ' + 'click here\n \n\n')