From 6e2bd49a81da6917a3e31da8ea4bdcdb174e1214 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Mon, 24 Aug 2009 01:16:49 -0300 Subject: [PATCH] Host header must include port number when port used for connecting is not default protocol port --- scrapy/core/downloader/webclient.py | 9 ++--- scrapy/tests/test_webclient.py | 54 +++++++++++++++-------------- 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 0ad5ab167..ead74ae79 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -13,9 +13,10 @@ def _parsed_url_args(parsed): host = parsed.hostname port = parsed.port scheme = parsed.scheme + netloc = parsed.netloc if port is None: port = 443 if scheme == 'https' else 80 - return scheme, host, port, path + return scheme, netloc, host, port, path def _parse(url): url = url.strip() @@ -82,16 +83,16 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): self.method = method self.body = body or None if parsedurl: - self.scheme, self.host, self.port, self.path = _parsed_url_args(parsedurl) + self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsedurl) else: - self.scheme, self.host, self.port, self.path = _parse(url) + self.scheme, self.netloc, self.host, self.port, self.path = _parse(url) self.timeout = timeout self.headers = Headers(headers or {}) self.deferred = defer.Deferred() # set Host header based on url - self.headers.setdefault('Host', self.host) + self.headers.setdefault('Host', self.netloc) # set Content-Length based len of body if self.body is not None: diff --git a/scrapy/tests/test_webclient.py b/scrapy/tests/test_webclient.py index e7a8f7dac..fa0799657 100644 --- a/scrapy/tests/test_webclient.py +++ b/scrapy/tests/test_webclient.py @@ -21,36 +21,37 @@ class ParseUrlTestCase(unittest.TestCase): def _parse(self, url): f = client.ScrapyHTTPClientFactory(url) - return (f.scheme, f.host, f.port, f.path) + return (f.scheme, f.netloc, f.host, f.port, f.path) def testParse(self): + lip = '127.0.0.1' tests = ( - ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', '127.0.0.1', 80, '/?c=v&c2=v2')), - ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', '127.0.0.1', 80, '/?c=v&c2=v2')), - ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', '127.0.0.1', 80, '/foo?c=v&c2=v2')), - ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', '127.0.0.1', 100, '/?c=v&c2=v2')), - ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', '127.0.0.1', 100, '/?c=v&c2=v2')), - ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', '127.0.0.1', 100, '/foo?c=v&c2=v2')), + ("http://127.0.0.1?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), + ("http://127.0.0.1/?c=v&c2=v2#fragment", ('http', lip, lip, 80, '/?c=v&c2=v2')), + ("http://127.0.0.1/foo?c=v&c2=v2#frag", ('http', lip, lip, 80, '/foo?c=v&c2=v2')), + ("http://127.0.0.1:100?c=v&c2=v2#fragment", ('http', lip+':100', lip, 100, '/?c=v&c2=v2')), + ("http://127.0.0.1:100/?c=v&c2=v2#frag", ('http', lip+':100', lip, 100, '/?c=v&c2=v2')), + ("http://127.0.0.1:100/foo?c=v&c2=v2#frag", ('http', lip+':100', lip, 100, '/foo?c=v&c2=v2')), - ("http://127.0.0.1", ('http', '127.0.0.1', 80, '/')), - ("http://127.0.0.1/", ('http', '127.0.0.1', 80, '/')), - ("http://127.0.0.1/foo", ('http', '127.0.0.1', 80, '/foo')), - ("http://127.0.0.1?param=value", ('http', '127.0.0.1', 80, '/?param=value')), - ("http://127.0.0.1/?param=value", ('http', '127.0.0.1', 80, '/?param=value')), - ("http://127.0.0.1:12345/foo", ('http', '127.0.0.1', 12345, '/foo')), - ("http://spam:12345/foo", ('http', 'spam', 12345, '/foo')), - ("http://spam.scrapytest.org/foo", ('http', 'spam.scrapytest.org', 80, '/foo')), + ("http://127.0.0.1", ('http', lip, lip, 80, '/')), + ("http://127.0.0.1/", ('http', lip, lip, 80, '/')), + ("http://127.0.0.1/foo", ('http', lip, lip, 80, '/foo')), + ("http://127.0.0.1?param=value", ('http', lip, lip, 80, '/?param=value')), + ("http://127.0.0.1/?param=value", ('http', lip, lip, 80, '/?param=value')), + ("http://127.0.0.1:12345/foo", ('http', lip+':12345', lip, 12345, '/foo')), + ("http://spam:12345/foo", ('http', 'spam:12345', 'spam', 12345, '/foo')), + ("http://spam.test.org/foo", ('http', 'spam.test.org', 'spam.test.org', 80, '/foo')), - ("https://127.0.0.1/foo", ('https', '127.0.0.1', 443, '/foo')), - ("https://127.0.0.1/?param=value", ('https', '127.0.0.1', 443, '/?param=value')), - ("https://127.0.0.1:12345/", ('https', '127.0.0.1', 12345, '/')), + ("https://127.0.0.1/foo", ('https', lip, lip, 443, '/foo')), + ("https://127.0.0.1/?param=value", ('https', lip, lip, 443, '/?param=value')), + ("https://127.0.0.1:12345/", ('https', lip+':12345', lip, 12345, '/')), - ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 80, '/foo')), - ("http://egg:7890 ", ('http', 'egg', 7890, '/')), - ) + ("http://scrapytest.org/foo ", ('http', 'scrapytest.org', 'scrapytest.org', 80, '/foo')), + ("http://egg:7890 ", ('http', 'egg:7890', 'egg', 7890, '/')), + ) for url, test in tests: - self.assertEquals(self._parse(url), test, url) + self.assertEquals(client._parse(url), test, url) def test_externalUnicodeInterference(self): """ @@ -61,8 +62,9 @@ class ParseUrlTestCase(unittest.TestCase): badInput = u'http://example.com/path' goodInput = badInput.encode('ascii') urlparse(badInput) - scheme, host, port, path = self._parse(goodInput) + scheme, netloc, host, port, path = self._parse(goodInput) self.assertTrue(isinstance(scheme, str)) + self.assertTrue(isinstance(netloc, str)) self.assertTrue(isinstance(host, str)) self.assertTrue(isinstance(path, str)) self.assertTrue(isinstance(port, int)) @@ -204,7 +206,7 @@ class WebClientTestCase(unittest.TestCase): # if we pass Host header explicitly, it should be used, otherwise # it should extract from url return defer.gatherResults([ - client.getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1"), + client.getPage(self.getURL("host")).addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno), client.getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(self.assertEquals, "www.example.com")]) @@ -238,7 +240,7 @@ class WebClientTestCase(unittest.TestCase): called back with the contents of the page. """ d = client.getPage(self.getURL("host"), timeout=100) - d.addCallback(self.assertEquals, "127.0.0.1") + d.addCallback(self.assertEquals, "127.0.0.1:%d" % self.portno) return d @@ -271,7 +273,7 @@ class WebClientTestCase(unittest.TestCase): def testFactoryInfo(self): url = self.getURL('file') - scheme, host, port, path = client._parse(url) + scheme, netloc, host, port, path = client._parse(url) factory = client.ScrapyHTTPClientFactory(url) reactor.connectTCP(host, port, factory) return factory.deferred.addCallback(self._cbFactoryInfo, factory)