From e5fb6094384f8d96e8a677b5fd9f2e6bfe2b97d3 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Thu, 14 Jan 2016 14:45:02 +0300
Subject: [PATCH] make ScrapyHTTPClientFactory comply to twisted
 HTTPClientFactory protocol - use bytes (encoding are likely wrong at this
 stage)

---
 scrapy/core/downloader/webclient.py | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index ba7bd798c..841322bdd 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -12,13 +12,15 @@ from scrapy.responsetypes import responsetypes
 
 
 def _parsed_url_args(parsed):
+    b = lambda x: to_bytes(x, encoding='ascii')
     path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
-    host = parsed.hostname
+    path = to_bytes(path)  # FIXME
+    host = b(parsed.hostname)  # FIXME
     port = parsed.port
-    scheme = parsed.scheme
-    netloc = parsed.netloc
+    scheme = b(parsed.scheme)
+    netloc = b(parsed.netloc)  # FIXME - host + port
     if port is None:
-        port = 443 if scheme == 'https' else 80
+        port = 443 if scheme == b'https' else 80
     return scheme, netloc, host, port, path
 
 
@@ -36,11 +38,7 @@ class ScrapyHTTPPageGetter(HTTPClient):
         self.headers = Headers() # bucket for response headers
 
         # Method command
-        self.sendCommand(
-            to_bytes(self.factory.method, encoding='ascii'),
-            # XXX - do we need to percent-encode path somewhere?
-            # https://en.wikipedia.org/wiki/Percent-encoding#Character_data
-            to_bytes(self.factory.path))
+        self.sendCommand(self.factory.method, self.factory.path)
         # Headers
         for key, values in self.factory.headers.items():
             for value in values:
@@ -96,8 +94,10 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
     afterFoundGet = False
 
     def __init__(self, request, timeout=180):
-        self.url = urldefrag(request.url)[0]
-        self.method = request.method
+        self._url = urldefrag(request.url)[0]
+        # converting to bytes to comply to Twisted interface
+        self.url = to_bytes(self._url)  # FIXME
+        self.method = to_bytes(request.method, encoding='ascii')
         self.body = request.body or None
         self.headers = Headers(request.headers)
         self.response_headers = None
@@ -131,11 +131,11 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
         request.meta['download_latency'] = self.headers_time-self.start_time
         status = int(self.status)
         headers = Headers(self.response_headers)
-        respcls = responsetypes.from_args(headers=headers, url=self.url)
+        respcls = responsetypes.from_args(headers=headers, url=self._url)
         # XXX - scrapy response stores body as bytes,
         # but maybe it makes sense to be able to store unicode?
         body = to_bytes(body)
-        return respcls(url=self.url, status=status, headers=headers, body=body)
+        return respcls(url=self._url, status=status, headers=headers, body=body)
 
     def _set_connection_attributes(self, request):
         parsed = urlparse_cached(request)