From f3889b0bce84cbd46852799a7bc95128c4c3b2d5 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 11:41:49 +0300 Subject: [PATCH] py3 compat: encode delimiter, method and path in ScrapyHTTPPageGetter --- scrapy/core/downloader/webclient.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index add5576ef..c335939d0 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -7,6 +7,7 @@ from twisted.internet import defer from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached +from scrapy.utils.python import to_bytes from scrapy.responsetypes import responsetypes @@ -29,13 +30,17 @@ def _parse(url): class ScrapyHTTPPageGetter(HTTPClient): - delimiter = '\n' + delimiter = b'\n' def connectionMade(self): self.headers = Headers() # bucket for response headers # Method command - self.sendCommand(self.factory.method, self.factory.path) + self.sendCommand( + to_bytes(self.factory.method, encoding='ascii'), + # XXX - do we need to percent-encode path somewhere? + # https://en.wikipedia.org/wiki/Percent-encoding#Character_data + to_bytes(self.factory.path)) # Headers for key, values in self.factory.headers.items(): for value in values: