[httpcompression] add support for br - brotli content encoding

requirements.txt

@@ -8,3 +8,4 @@ six>=1.5.2
 PyDispatcher>=2.0.5
 service_identity
 parsel>=1.1
+brotlipy==0.6

scrapy/downloadermiddlewares/httpcompression.py

@@ -1,5 +1,7 @@
 import zlib
 
+import brotli
+
 from scrapy.utils.gz import gunzip, is_gzipped
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
@@ -17,7 +19,7 @@ class HttpCompressionMiddleware(object):
         return cls()
 
     def process_request(self, request, spider):
-        request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
+        request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br')
 
     def process_response(self, request, response, spider):
 
@@ -55,5 +57,7 @@ class HttpCompressionMiddleware(object):
                 # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
                 # http://www.gzip.org/zlib/zlib_faq.html#faq38
                 body = zlib.decompress(body, -15)
+        if encoding == b"br":
+            body = brotli.decompress(body)
         return body
 

tests/test_downloadermiddleware_httpcompression.py

@@ -17,6 +17,7 @@ FORMAT = {
         'x-gzip': ('html-gzip.bin', 'gzip'),
         'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
         'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
+        'br': ('html-br.bin', 'br')
         }
 
 class HttpCompressionTest(TestCase):
@@ -50,7 +51,8 @@ class HttpCompressionTest(TestCase):
         request = Request('http://scrapytest.org')
         assert 'Accept-Encoding' not in request.headers
         self.mw.process_request(request, self.spider)
-        self.assertEqual(request.headers.get('Accept-Encoding'), b'gzip,deflate')
+        self.assertEqual(request.headers.get('Accept-Encoding'),
+                         b'gzip,deflate,br')
 
     def test_process_response_gzip(self):
         response = self._getresponse('gzip')
@@ -62,6 +64,15 @@ class HttpCompressionTest(TestCase):
         assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
 
+    def test_process_response_br(self):
+        response = self._getresponse('br')
+        request = response.request
+        self.assertEqual(response.headers['Content-Encoding'], b'br')
+        newresponse = self.mw.process_response(request, response, self.spider)
+        assert newresponse is not response
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert 'Content-Encoding' not in newresponse.headers
+
     def test_process_response_rawdeflate(self):
         response = self._getresponse('rawdeflate')
         request = response.request