diff --git a/requirements.txt b/requirements.txt index f92603d3d..362d05013 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ six>=1.5.2 PyDispatcher>=2.0.5 service_identity parsel>=1.1 +brotlipy==0.6 diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index bcf20f10c..9202fd8da 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,5 +1,7 @@ import zlib +import brotli + from scrapy.utils.gz import gunzip, is_gzipped from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes @@ -17,7 +19,7 @@ class HttpCompressionMiddleware(object): return cls() def process_request(self, request, spider): - request.headers.setdefault('Accept-Encoding', 'gzip,deflate') + request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br') def process_response(self, request, response, spider): @@ -55,5 +57,7 @@ class HttpCompressionMiddleware(object): # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx # http://www.gzip.org/zlib/zlib_faq.html#faq38 body = zlib.decompress(body, -15) + if encoding == b"br": + body = brotli.decompress(body) return body diff --git a/tests/sample_data/compressed/html-br.bin b/tests/sample_data/compressed/html-br.bin new file mode 100644 index 000000000..c7eea4bb8 Binary files /dev/null and b/tests/sample_data/compressed/html-br.bin differ diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index b2426946d..c47de54ed 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -17,6 +17,7 @@ FORMAT = { 'x-gzip': ('html-gzip.bin', 'gzip'), 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), + 'br': ('html-br.bin', 'br') } class HttpCompressionTest(TestCase): @@ -50,7 +51,8 @@ class HttpCompressionTest(TestCase): request = Request('http://scrapytest.org') assert 'Accept-Encoding' not in request.headers self.mw.process_request(request, self.spider) - self.assertEqual(request.headers.get('Accept-Encoding'), b'gzip,deflate') + self.assertEqual(request.headers.get('Accept-Encoding'), + b'gzip,deflate,br') def test_process_response_gzip(self): response = self._getresponse('gzip') @@ -62,6 +64,15 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b'