diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index d81093a9f..82cf507f7 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -78,7 +78,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
for it.
"""
- _responseMatcher = re.compile('HTTP/1\.. 200')
+ _responseMatcher = re.compile(b'HTTP/1\.. 200')
def __init__(self, reactor, host, port, proxyConf, contextFactory,
timeout=30, bindAddress=None):
@@ -92,11 +92,15 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
def requestTunnel(self, protocol):
"""Asks the proxy to open a tunnel."""
- tunnelReq = 'CONNECT %s:%s HTTP/1.1\r\n' % (self._tunneledHost,
- self._tunneledPort)
+ tunnelReq = (
+ b'CONNECT ' +
+ to_bytes(self._tunneledHost, encoding='ascii') + b':' +
+ to_bytes(str(self._tunneledPort)) +
+ b' HTTP/1.1\r\n')
if self._proxyAuthHeader:
- tunnelReq += 'Proxy-Authorization: %s\r\n' % self._proxyAuthHeader
- tunnelReq += '\r\n'
+ tunnelReq += \
+ b'Proxy-Authorization: ' + self._proxyAuthHeader + b'\r\n'
+ tunnelReq += b'\r\n'
protocol.transport.write(tunnelReq)
self._protocolDataReceived = protocol.dataReceived
protocol.dataReceived = self.processProxyResponse
@@ -202,7 +206,7 @@ class ScrapyAgent(object):
agent = self._get_agent(request, timeout)
# request details
- url = to_bytes(urldefrag(request.url)[0])
+ url = urldefrag(request.url)[0]
method = to_bytes(request.method)
headers = TxHeaders(request.headers)
if isinstance(agent, self._TunnelingAgent):
@@ -210,7 +214,8 @@ class ScrapyAgent(object):
bodyproducer = _RequestBodyProducer(request.body) if request.body else None
start_time = time()
- d = agent.request(method, url, headers, bodyproducer)
+ d = agent.request(
+ method, to_bytes(url, encoding='ascii'), headers, bodyproducer)
# set download latency
d.addCallback(self._cb_latency, request, start_time)
# response body is ready to be consumed
@@ -263,10 +268,8 @@ class ScrapyAgent(object):
txresponse, body, flags = result
status = int(txresponse.code)
headers = Headers(txresponse.headers.getAllRawHeaders())
- url = to_unicode(url)
respcls = responsetypes.from_args(headers=headers, url=url)
- return respcls(
- url=url, status=status, headers=headers, body=body, flags=flags)
+ return respcls(url=url, status=status, headers=headers, body=body, flags=flags)
@implementer(IBodyProducer)
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index d2cdd6f98..9bcc51943 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -12,18 +12,26 @@ from scrapy.responsetypes import responsetypes
def _parsed_url_args(parsed):
+ # Assume parsed is urlparse-d from Request.url,
+ # which was passed via safe_url_string and is ascii-only.
+ b = lambda s: to_bytes(s, encoding='ascii')
path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
- path = to_bytes(path)
- host = to_bytes(parsed.hostname)
+ path = b(path)
+ host = b(parsed.hostname)
port = parsed.port
- scheme = to_bytes(parsed.scheme, encoding='ascii')
- netloc = to_bytes(parsed.netloc)
+ scheme = b(parsed.scheme)
+ netloc = b(parsed.netloc)
if port is None:
port = 443 if scheme == b'https' else 80
return scheme, netloc, host, port, path
def _parse(url):
+ """ Return tuple of (scheme, netloc, host, port, path),
+ all in bytes except for port which is int.
+ Assume url is from Request.url, which was passed via safe_url_string
+ and is ascii-only.
+ """
url = url.strip()
parsed = urlparse(url)
return _parsed_url_args(parsed)
@@ -66,7 +74,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
def handleResponse(self, response):
if self.factory.method.upper() == b'HEAD':
- self.factory.page('')
+ self.factory.page(b'')
elif self.length is not None and self.length > 0:
self.factory.noPage(self._connection_lost_reason)
else:
@@ -95,7 +103,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
def __init__(self, request, timeout=180):
self._url = urldefrag(request.url)[0]
# converting to bytes to comply to Twisted interface
- self.url = to_bytes(self._url)
+ self.url = to_bytes(self._url, encoding='ascii')
self.method = to_bytes(request.method, encoding='ascii')
self.body = request.body or None
self.headers = Headers(request.headers)
@@ -131,7 +139,6 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
status = int(self.status)
headers = Headers(self.response_headers)
respcls = responsetypes.from_args(headers=headers, url=self._url)
- body = to_bytes(body)
return respcls(url=self._url, status=status, headers=headers, body=body)
def _set_connection_attributes(self, request):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 84d1aa191..59320597e 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -119,7 +119,7 @@ class HttpTestCase(unittest.TestCase):
r.putChild(b"broken", BrokenDownloadResource())
self.site = server.Site(r, timeout=None)
self.wrapper = WrappingFactory(self.site)
- self.host = '127.0.0.1'
+ self.host = 'localhost'
if self.scheme == 'https':
self.port = reactor.listenSSL(
0, self.wrapper, ssl_context_factory(), interface=self.host)
@@ -168,6 +168,9 @@ class HttpTestCase(unittest.TestCase):
@defer.inlineCallbacks
def test_timeout_download_from_spider(self):
+ if self.scheme == 'https':
+ raise unittest.SkipTest(
+ 'test_timeout_download_from_spider skipped under https')
spider = Spider('foo')
meta = {'download_timeout': 0.2}
# client connects but no data is received
@@ -181,7 +184,8 @@ class HttpTestCase(unittest.TestCase):
def test_host_header_not_in_request_headers(self):
def _test(response):
- self.assertEquals(response.body, to_bytes('127.0.0.1:%d' % self.portno))
+ self.assertEquals(
+ response.body, to_bytes('%s:%d' % (self.host, self.portno)))
self.assertEquals(request.headers, {})
request = Request(self.getURL('host'))
@@ -221,8 +225,6 @@ class Http10TestCase(HttpTestCase):
class Https10TestCase(Http10TestCase):
scheme = 'https'
- def test_timeout_download_from_spider(self):
- raise unittest.SkipTest("test_timeout_download_from_spider skipped under https")
class Http11TestCase(HttpTestCase):
@@ -273,6 +275,10 @@ class Http11TestCase(HttpTestCase):
return d
+class Https11TestCase(Http11TestCase):
+ scheme = 'https'
+
+
class Http11MockServerTestCase(unittest.TestCase):
"""HTTP 1.1 test case with MockServer"""
if twisted_version < (11, 1, 0):
@@ -406,6 +412,17 @@ class Http11ProxyTestCase(HttpProxyTestCase):
if twisted_version < (11, 1, 0):
skip = 'HTTP1.1 not supported in twisted < 11.1.0'
+ @defer.inlineCallbacks
+ def test_download_with_proxy_https_timeout(self):
+ """ Test TunnelingTCP4ClientEndpoint """
+ http_proxy = self.getURL('')
+ domain = 'https://no-such-domain.nosuch'
+ request = Request(
+ domain, meta={'proxy': http_proxy, 'download_timeout': 0.2})
+ d = self.download_request(request, Spider('foo'))
+ timeout = yield self.assertFailure(d, error.TimeoutError)
+ self.assertIn(domain, timeout.osError)
+
class HttpDownloadHandlerMock(object):
def __init__(self, settings):
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 412e10c89..9b5beda4c 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -7,7 +7,7 @@ import six
from six.moves.urllib.parse import urlparse
from twisted.trial import unittest
-from twisted.web import server, static, error, util
+from twisted.web import server, static, util, resource
from twisted.internet import reactor, defer
from twisted.test.proto_helpers import StringTransport
from twisted.python.filepath import FilePath
@@ -18,14 +18,14 @@ from scrapy.http import Request, Headers
from scrapy.utils.python import to_bytes, to_unicode
-def getPage(url, contextFactory=None, *args, **kwargs):
+def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
"""Adapted version of twisted.web.client.getPage"""
def _clientfactory(url, *args, **kwargs):
url = to_unicode(url)
timeout = kwargs.pop('timeout', 0)
f = client.ScrapyHTTPClientFactory(
Request(url, *args, **kwargs), timeout=timeout)
- f.deferred.addCallback(lambda r: r.body)
+ f.deferred.addCallback(response_transform or (lambda r: r.body))
return f
from twisted.web.client import _makeGetterFactory
@@ -78,16 +78,17 @@ class ParseUrlTestCase(unittest.TestCase):
elements of its return tuple, even when passed an URL which has
previously been passed to L{urlparse} as a C{unicode} string.
"""
- goodInput = u'http://example.com/path'
- badInput = goodInput.encode('ascii')
- if six.PY2:
- goodInput, badInput = badInput, goodInput
- urlparse(badInput)
+ if not six.PY2:
+ raise unittest.SkipTest(
+ "Applies only to Py2, as urls can be ONLY unicode on Py3")
+ badInput = u'http://example.com/path'
+ goodInput = badInput.encode('ascii')
+ self._parse(badInput) # cache badInput in urlparse_cached
scheme, netloc, host, port, path = self._parse(goodInput)
- self.assertTrue(isinstance(scheme, bytes))
- self.assertTrue(isinstance(netloc, bytes))
- self.assertTrue(isinstance(host, bytes))
- self.assertTrue(isinstance(path, bytes))
+ self.assertTrue(isinstance(scheme, str))
+ self.assertTrue(isinstance(netloc, str))
+ self.assertTrue(isinstance(host, str))
+ self.assertTrue(isinstance(path, str))
self.assertTrue(isinstance(port, int))
@@ -213,6 +214,16 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
ErrorResource, NoLengthResource, HostHeaderResource, \
PayloadResource, BrokenDownloadResource
+
+class EncodingResource(resource.Resource):
+ out_encoding = 'cp1251'
+
+ def render(self, request):
+ body = to_unicode(request.content.read())
+ request.setHeader(b'content-encoding', self.out_encoding)
+ return body.encode(self.out_encoding)
+
+
class WebClientTestCase(unittest.TestCase):
def _listen(self, site):
return reactor.listenTCP(0, site, interface="127.0.0.1")
@@ -229,6 +240,7 @@ class WebClientTestCase(unittest.TestCase):
r.putChild(b"host", HostHeaderResource())
r.putChild(b"payload", PayloadResource())
r.putChild(b"broken", BrokenDownloadResource())
+ r.putChild(b"encoding", EncodingResource())
self.site = server.Site(r, timeout=None)
self.wrapper = WrappingFactory(self.site)
self.port = self._listen(self.wrapper)
@@ -338,3 +350,17 @@ class WebClientTestCase(unittest.TestCase):
b'\n\n \n \n'
b' \n \n '
b'click here\n \n\n')
+
+ def test_Encoding(self):
+ """ Test that non-standart body encoding matches
+ Content-Encoding header """
+ body = b'\xd0\x81\xd1\x8e\xd0\xaf'
+ return getPage(
+ self.getURL('encoding'), body=body, response_transform=lambda r: r)\
+ .addCallback(self._check_Encoding, body)
+
+ def _check_Encoding(self, response, original_body):
+ content_encoding = to_unicode(response.headers[b'Content-Encoding'])
+ self.assertEquals(content_encoding, EncodingResource.out_encoding)
+ self.assertEquals(
+ response.body.decode(content_encoding), to_unicode(original_body))