Merge branch 'py3-webclient' into py3-http-downloaders

scrapy/core/downloader/webclient.py

@@ -12,18 +12,26 @@ from scrapy.responsetypes import responsetypes
 
 
 def _parsed_url_args(parsed):
+    # Assume parsed is urlparse-d from Request.url,
+    # which was passed via safe_url_string and is ascii-only.
+    b = lambda s: to_bytes(s, encoding='ascii')
     path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
-    path = to_bytes(path)
-    host = to_bytes(parsed.hostname)
+    path = b(path)
+    host = b(parsed.hostname)
     port = parsed.port
-    scheme = to_bytes(parsed.scheme, encoding='ascii')
-    netloc = to_bytes(parsed.netloc)
+    scheme = b(parsed.scheme)
+    netloc = b(parsed.netloc)
     if port is None:
         port = 443 if scheme == b'https' else 80
     return scheme, netloc, host, port, path
 
 
 def _parse(url):
+    """ Return tuple of (scheme, netloc, host, port, path),
+    all in bytes except for port which is int.
+    Assume url is from Request.url, which was passed via safe_url_string
+    and is ascii-only.
+    """
     url = url.strip()
     parsed = urlparse(url)
     return _parsed_url_args(parsed)
@@ -66,7 +74,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
 
     def handleResponse(self, response):
         if self.factory.method.upper() == b'HEAD':
-            self.factory.page('')
+            self.factory.page(b'')
         elif self.length is not None and self.length > 0:
             self.factory.noPage(self._connection_lost_reason)
         else:
@@ -95,7 +103,7 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
     def __init__(self, request, timeout=180):
         self._url = urldefrag(request.url)[0]
         # converting to bytes to comply to Twisted interface
-        self.url = to_bytes(self._url)
+        self.url = to_bytes(self._url, encoding='ascii')
         self.method = to_bytes(request.method, encoding='ascii')
         self.body = request.body or None
         self.headers = Headers(request.headers)
@@ -131,7 +139,6 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
         status = int(self.status)
         headers = Headers(self.response_headers)
         respcls = responsetypes.from_args(headers=headers, url=self._url)
-        body = to_bytes(body)
         return respcls(url=self._url, status=status, headers=headers, body=body)
 
     def _set_connection_attributes(self, request):

tests/test_webclient.py

@@ -7,7 +7,7 @@ import six
 from six.moves.urllib.parse import urlparse
 
 from twisted.trial import unittest
-from twisted.web import server, static, error, util
+from twisted.web import server, static, util, resource
 from twisted.internet import reactor, defer
 from twisted.test.proto_helpers import StringTransport
 from twisted.python.filepath import FilePath
@@ -18,14 +18,14 @@ from scrapy.http import Request, Headers
 from scrapy.utils.python import to_bytes, to_unicode
 
 
-def getPage(url, contextFactory=None, *args, **kwargs):
+def getPage(url, contextFactory=None, r_transform=None, *args, **kwargs):
     """Adapted version of twisted.web.client.getPage"""
     def _clientfactory(url, *args, **kwargs):
         url = to_unicode(url)
         timeout = kwargs.pop('timeout', 0)
         f = client.ScrapyHTTPClientFactory(
             Request(url, *args, **kwargs), timeout=timeout)
-        f.deferred.addCallback(lambda r: r.body)
+        f.deferred.addCallback(r_transform or (lambda r: r.body))
         return f
 
     from twisted.web.client import _makeGetterFactory
@@ -213,6 +213,16 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
         ErrorResource, NoLengthResource, HostHeaderResource, \
         PayloadResource, BrokenDownloadResource
 
+
+class EncodingResource(resource.Resource):
+    out_encoding = 'cp1251'
+
+    def render(self, request):
+        body = to_unicode(request.content.read())
+        request.setHeader(b'content-encoding', self.out_encoding)
+        return body.encode(self.out_encoding)
+
+
 class WebClientTestCase(unittest.TestCase):
     def _listen(self, site):
         return reactor.listenTCP(0, site, interface="127.0.0.1")
@@ -229,6 +239,7 @@ class WebClientTestCase(unittest.TestCase):
         r.putChild(b"host", HostHeaderResource())
         r.putChild(b"payload", PayloadResource())
         r.putChild(b"broken", BrokenDownloadResource())
+        r.putChild(b"encoding", EncodingResource())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
         self.port = self._listen(self.wrapper)
@@ -338,3 +349,17 @@ class WebClientTestCase(unittest.TestCase):
                 b'\n<html>\n    <head>\n        <meta http-equiv="refresh" content="0;URL=/file">\n'
                 b'    </head>\n    <body bgcolor="#FFFFFF" text="#000000">\n    '
                 b'<a href="/file">click here</a>\n    </body>\n</html>\n')
+
+    def test_Encoding(self):
+        """ Test that non-standart body encoding matches
+        Content-Encoding header """
+        body = b'\xd0\x81\xd1\x8e\xd0\xaf'
+        return getPage(
+            self.getURL('encoding'), body=body, r_transform=lambda r: r)\
+            .addCallback(self._check_Encoding, body)
+
+    def _check_Encoding(self, response, original_body):
+        content_encoding = to_unicode(response.headers[b'Content-Encoding'])
+        self.assertEquals(content_encoding, EncodingResource.out_encoding)
+        self.assertEquals(
+            response.body.decode(content_encoding), to_unicode(original_body))