From 0b9336418ef40ca95052ebbaa02f12953e165115 Mon Sep 17 00:00:00 2001
From: Konstantin Lopuhin <kostia.lopuhin@gmail.com>
Date: Mon, 18 Jan 2016 16:43:58 +0300
Subject: [PATCH] py3: port compression downloader middleware and tests

---
 .../downloadermiddlewares/httpcompression.py  |  8 +++----
 tests/py3-ignores.txt                         |  1 -
 ...st_downloadermiddleware_httpcompression.py | 24 +++++++++----------
 3 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 719507396..7ab304c17 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -9,13 +9,13 @@ from scrapy.exceptions import NotConfigured
 class HttpCompressionMiddleware(object):
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
-    
+
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('COMPRESSION_ENABLED'):
             raise NotConfigured
         return cls()
-    
+
     def process_request(self, request, spider):
         request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
 
@@ -39,10 +39,10 @@ class HttpCompressionMiddleware(object):
         return response
 
     def _decode(self, body, encoding):
-        if encoding == 'gzip' or encoding == 'x-gzip':
+        if encoding == b'gzip' or encoding == b'x-gzip':
             body = gunzip(body)
 
-        if encoding == 'deflate':
+        if encoding == b'deflate':
             try:
                 body = zlib.decompress(body)
             except zlib.error:
diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt
index 2a9f06c8c..dbf63f0f5 100644
--- a/tests/py3-ignores.txt
+++ b/tests/py3-ignores.txt
@@ -4,7 +4,6 @@ tests/test_command_shell.py
 tests/test_exporters.py
 tests/test_linkextractors_deprecated.py
 tests/test_crawl.py
-tests/test_downloadermiddleware_httpcompression.py
 tests/test_downloadermiddleware_httpproxy.py
 tests/test_downloadermiddleware.py
 tests/test_downloadermiddleware_retry.py
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index a18994ef3..2e6e47fef 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -50,46 +50,46 @@ class HttpCompressionTest(TestCase):
         request = Request('http://scrapytest.org')
         assert 'Accept-Encoding' not in request.headers
         self.mw.process_request(request, self.spider)
-        self.assertEqual(request.headers.get('Accept-Encoding'), 'gzip,deflate')
+        self.assertEqual(request.headers.get('Accept-Encoding'), b'gzip,deflate')
 
     def test_process_response_gzip(self):
         response = self._getresponse('gzip')
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], 'gzip')
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
-        assert newresponse.body.startswith('<!DOCTYPE')
+        assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
 
     def test_process_response_rawdeflate(self):
         response = self._getresponse('rawdeflate')
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], 'deflate')
+        self.assertEqual(response.headers['Content-Encoding'], b'deflate')
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
-        assert newresponse.body.startswith('<!DOCTYPE')
+        assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
 
     def test_process_response_zlibdelate(self):
         response = self._getresponse('zlibdeflate')
         request = response.request
 
-        self.assertEqual(response.headers['Content-Encoding'], 'deflate')
+        self.assertEqual(response.headers['Content-Encoding'], b'deflate')
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
-        assert newresponse.body.startswith('<!DOCTYPE')
+        assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
 
     def test_process_response_plain(self):
-        response = Response('http://scrapytest.org', body='<!DOCTYPE...')
+        response = Response('http://scrapytest.org', body=b'<!DOCTYPE...')
         request = Request('http://scrapytest.org')
 
         assert not response.headers.get('Content-Encoding')
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is response
-        assert newresponse.body.startswith('<!DOCTYPE')
+        assert newresponse.body.startswith(b'<!DOCTYPE')
 
     def test_multipleencodings(self):
         response = self._getresponse('gzip')
@@ -97,7 +97,7 @@ class HttpCompressionTest(TestCase):
         request = response.request
         newresponse = self.mw.process_response(request, response, self.spider)
         assert newresponse is not response
-        self.assertEqual(newresponse.headers.getlist('Content-Encoding'), ['uuencode'])
+        self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode'])
 
     def test_process_response_encoding_inside_body(self):
         headers = {
@@ -142,5 +142,5 @@ class HttpCompressionTest(TestCase):
 
         newresponse = self.mw.process_response(request, response, self.spider)
         self.assertIs(newresponse, response)
-        self.assertEqual(response.headers['Content-Encoding'], 'gzip')
-        self.assertEqual(response.headers['Content-Type'], 'application/gzip')
+        self.assertEqual(response.headers['Content-Encoding'], b'gzip')
+        self.assertEqual(response.headers['Content-Type'], b'application/gzip')