Merge pull request #2535 from pawelmhm/brotli

[MRG+1] [httpcompression] add support for br - brotli content encoding

scrapy/downloadermiddlewares/httpcompression.py

@@ -6,10 +6,18 @@ from scrapy.responsetypes import responsetypes
 from scrapy.exceptions import NotConfigured
 
 
+ACCEPTED_ENCODINGS = [b'gzip', b'deflate']
+
+try:
+    import brotli
+    ACCEPTED_ENCODINGS.append(b'br')
+except ImportError:
+    pass
+
+
 class HttpCompressionMiddleware(object):
     """This middleware allows compressed (gzip, deflate) traffic to be
     sent/received from web sites"""
-
     @classmethod
     def from_crawler(cls, crawler):
         if not crawler.settings.getbool('COMPRESSION_ENABLED'):
@@ -17,7 +25,8 @@ class HttpCompressionMiddleware(object):
         return cls()
 
     def process_request(self, request, spider):
-        request.headers.setdefault('Accept-Encoding', 'gzip,deflate')
+        request.headers.setdefault('Accept-Encoding',
+                                   b",".join(ACCEPTED_ENCODINGS))
 
     def process_response(self, request, response, spider):
 
@@ -55,5 +64,6 @@ class HttpCompressionMiddleware(object):
                 # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
                 # http://www.gzip.org/zlib/zlib_faq.html#faq38
                 body = zlib.decompress(body, -15)
+        if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS:
+            body = brotli.decompress(body)
         return body
-

tests/requirements-py3.txt

@@ -8,3 +8,4 @@ botocore
 # optional for shell wrapper tests
 bpython
 ipython
+brotlipy

tests/requirements.txt

@@ -6,6 +6,7 @@ pytest==2.9.2
 pytest-twisted
 pytest-cov==2.2.1
 jmespath
+brotlipy
 testfixtures
 # optional for shell wrapper tests
 bpython

tests/test_downloadermiddleware_httpcompression.py

@@ -1,11 +1,12 @@
 from io import BytesIO
-from unittest import TestCase
-from os.path import join, abspath, dirname
+from unittest import TestCase, SkipTest
+from os.path import join
 from gzip import GzipFile
 
 from scrapy.spiders import Spider
 from scrapy.http import Response, Request, HtmlResponse
-from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware
+from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \
+    ACCEPTED_ENCODINGS
 from tests import tests_datadir
 from w3lib.encoding import resolve_encoding
 
@@ -17,8 +18,10 @@ FORMAT = {
         'x-gzip': ('html-gzip.bin', 'gzip'),
         'rawdeflate': ('html-rawdeflate.bin', 'deflate'),
         'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'),
+        'br': ('html-br.bin', 'br')
         }
 
+
 class HttpCompressionTest(TestCase):
 
     def setUp(self):
@@ -50,7 +53,8 @@ class HttpCompressionTest(TestCase):
         request = Request('http://scrapytest.org')
         assert 'Accept-Encoding' not in request.headers
         self.mw.process_request(request, self.spider)
-        self.assertEqual(request.headers.get('Accept-Encoding'), b'gzip,deflate')
+        self.assertEqual(request.headers.get('Accept-Encoding'),
+                         b','.join(ACCEPTED_ENCODINGS))
 
     def test_process_response_gzip(self):
         response = self._getresponse('gzip')
@@ -62,6 +66,19 @@ class HttpCompressionTest(TestCase):
         assert newresponse.body.startswith(b'<!DOCTYPE')
         assert 'Content-Encoding' not in newresponse.headers
 
+    def test_process_response_br(self):
+        try:
+            import brotli
+        except ImportError:
+            raise SkipTest("no brotli")
+        response = self._getresponse('br')
+        request = response.request
+        self.assertEqual(response.headers['Content-Encoding'], b'br')
+        newresponse = self.mw.process_response(request, response, self.spider)
+        assert newresponse is not response
+        assert newresponse.body.startswith(b"<!DOCTYPE")
+        assert 'Content-Encoding' not in newresponse.headers
+
     def test_process_response_rawdeflate(self):
         response = self._getresponse('rawdeflate')
         request = response.request