From d3c2b0cf7ec185d18464e85974ee7499acecad79 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 10:06:14 +0300 Subject: [PATCH 1/3] py3 webclient: I was mistaken about unicode body, revert conversion to bytes and fix HEAD response --- scrapy/core/downloader/webclient.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index d2cdd6f98..bbbb98f60 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -66,7 +66,7 @@ class ScrapyHTTPPageGetter(HTTPClient): def handleResponse(self, response): if self.factory.method.upper() == b'HEAD': - self.factory.page('') + self.factory.page(b'') elif self.length is not None and self.length > 0: self.factory.noPage(self._connection_lost_reason) else: @@ -131,7 +131,6 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self._url) - body = to_bytes(body) return respcls(url=self._url, status=status, headers=headers, body=body) def _set_connection_attributes(self, request): From 673df5e4161337136154eeccfc4e327d053dbd12 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 11:44:49 +0300 Subject: [PATCH 2/3] add webclient test - check that non-standart body encoding matches Content-Encoding header --- tests/test_webclient.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 412e10c89..3ee6c24c2 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -7,7 +7,7 @@ import six from six.moves.urllib.parse import urlparse from twisted.trial import unittest -from twisted.web import server, static, error, util +from twisted.web import server, static, util, resource from twisted.internet import reactor, defer from twisted.test.proto_helpers import StringTransport from twisted.python.filepath import FilePath @@ -18,14 +18,14 @@ from scrapy.http import Request, Headers from scrapy.utils.python import to_bytes, to_unicode -def getPage(url, contextFactory=None, *args, **kwargs): +def getPage(url, contextFactory=None, r_transform=None, *args, **kwargs): """Adapted version of twisted.web.client.getPage""" def _clientfactory(url, *args, **kwargs): url = to_unicode(url) timeout = kwargs.pop('timeout', 0) f = client.ScrapyHTTPClientFactory( Request(url, *args, **kwargs), timeout=timeout) - f.deferred.addCallback(lambda r: r.body) + f.deferred.addCallback(r_transform or (lambda r: r.body)) return f from twisted.web.client import _makeGetterFactory @@ -213,6 +213,16 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \ ErrorResource, NoLengthResource, HostHeaderResource, \ PayloadResource, BrokenDownloadResource + +class EncodingResource(resource.Resource): + out_encoding = 'cp1251' + + def render(self, request): + body = to_unicode(request.content.read()) + request.setHeader(b'content-encoding', self.out_encoding) + return body.encode(self.out_encoding) + + class WebClientTestCase(unittest.TestCase): def _listen(self, site): return reactor.listenTCP(0, site, interface="127.0.0.1") @@ -229,6 +239,7 @@ class WebClientTestCase(unittest.TestCase): r.putChild(b"host", HostHeaderResource()) r.putChild(b"payload", PayloadResource()) r.putChild(b"broken", BrokenDownloadResource()) + r.putChild(b"encoding", EncodingResource()) self.site = server.Site(r, timeout=None) self.wrapper = WrappingFactory(self.site) self.port = self._listen(self.wrapper) @@ -338,3 +349,17 @@ class WebClientTestCase(unittest.TestCase): b'\n\n \n \n' b' \n \n ' b'click here\n \n\n') + + def test_Encoding(self): + """ Test that non-standart body encoding matches + Content-Encoding header """ + body = b'\xd0\x81\xd1\x8e\xd0\xaf' + return getPage( + self.getURL('encoding'), body=body, r_transform=lambda r: r)\ + .addCallback(self._check_Encoding, body) + + def _check_Encoding(self, response, original_body): + content_encoding = to_unicode(response.headers[b'Content-Encoding']) + self.assertEquals(content_encoding, EncodingResource.out_encoding) + self.assertEquals( + response.body.decode(content_encoding), to_unicode(original_body)) From f2e2ff5e1f2eb1d11e18be248e27bef7fc9b4ef7 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Mon, 18 Jan 2016 12:20:01 +0300 Subject: [PATCH 3/3] py3: webclient assumes that urls come from Request.url and are ascii-only --- scrapy/core/downloader/webclient.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index bbbb98f60..9bcc51943 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -12,18 +12,26 @@ from scrapy.responsetypes import responsetypes def _parsed_url_args(parsed): + # Assume parsed is urlparse-d from Request.url, + # which was passed via safe_url_string and is ascii-only. + b = lambda s: to_bytes(s, encoding='ascii') path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, '')) - path = to_bytes(path) - host = to_bytes(parsed.hostname) + path = b(path) + host = b(parsed.hostname) port = parsed.port - scheme = to_bytes(parsed.scheme, encoding='ascii') - netloc = to_bytes(parsed.netloc) + scheme = b(parsed.scheme) + netloc = b(parsed.netloc) if port is None: port = 443 if scheme == b'https' else 80 return scheme, netloc, host, port, path def _parse(url): + """ Return tuple of (scheme, netloc, host, port, path), + all in bytes except for port which is int. + Assume url is from Request.url, which was passed via safe_url_string + and is ascii-only. + """ url = url.strip() parsed = urlparse(url) return _parsed_url_args(parsed) @@ -95,7 +103,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): def __init__(self, request, timeout=180): self._url = urldefrag(request.url)[0] # converting to bytes to comply to Twisted interface - self.url = to_bytes(self._url) + self.url = to_bytes(self._url, encoding='ascii') self.method = to_bytes(request.method, encoding='ascii') self.body = request.body or None self.headers = Headers(request.headers)