From 73ff87c1dc05e98104f854d16792e302625c5e98 Mon Sep 17 00:00:00 2001 From: Konstantin Lopuhin Date: Thu, 14 Jan 2016 12:03:08 +0300 Subject: [PATCH] decode body from utf-8, as scrapy stores body as bytes, and twisted has already converted to unicode --- scrapy/core/downloader/webclient.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index c335939d0..ba7bd798c 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -132,6 +132,9 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): status = int(self.status) headers = Headers(self.response_headers) respcls = responsetypes.from_args(headers=headers, url=self.url) + # XXX - scrapy response stores body as bytes, + # but maybe it makes sense to be able to store unicode? + body = to_bytes(body) return respcls(url=self.url, status=status, headers=headers, body=body) def _set_connection_attributes(self, request):