diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index bcf20f10c..19d6345e4 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -6,10 +6,18 @@ from scrapy.responsetypes import responsetypes from scrapy.exceptions import NotConfigured +ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] + +try: + import brotli + ACCEPTED_ENCODINGS.append(b'br') +except ImportError: + pass + + class HttpCompressionMiddleware(object): """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): @@ -17,7 +25,8 @@ class HttpCompressionMiddleware(object): return cls() def process_request(self, request, spider): - request.headers.setdefault('Accept-Encoding', 'gzip,deflate') + request.headers.setdefault('Accept-Encoding', + b",".join(ACCEPTED_ENCODINGS)) def process_response(self, request, response, spider): @@ -55,5 +64,6 @@ class HttpCompressionMiddleware(object): # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx # http://www.gzip.org/zlib/zlib_faq.html#faq38 body = zlib.decompress(body, -15) + if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: + body = brotli.decompress(body) return body - diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt index ed189c66c..51a25f5e5 100644 --- a/tests/requirements-py3.txt +++ b/tests/requirements-py3.txt @@ -8,3 +8,4 @@ botocore # optional for shell wrapper tests bpython ipython +brotlipy diff --git a/tests/requirements.txt b/tests/requirements.txt index 9d0c3c996..c1576a2e7 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,6 +6,7 @@ pytest==2.9.2 pytest-twisted pytest-cov==2.2.1 jmespath +brotlipy testfixtures # optional for shell wrapper tests bpython diff --git a/tests/sample_data/compressed/html-br.bin b/tests/sample_data/compressed/html-br.bin new file mode 100644 index 000000000..c7eea4bb8 Binary files /dev/null and b/tests/sample_data/compressed/html-br.bin differ diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index b2426946d..7924fb3b5 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,11 +1,12 @@ from io import BytesIO -from unittest import TestCase -from os.path import join, abspath, dirname +from unittest import TestCase, SkipTest +from os.path import join from gzip import GzipFile from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse -from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware +from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \ + ACCEPTED_ENCODINGS from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -17,8 +18,10 @@ FORMAT = { 'x-gzip': ('html-gzip.bin', 'gzip'), 'rawdeflate': ('html-rawdeflate.bin', 'deflate'), 'zlibdeflate': ('html-zlibdeflate.bin', 'deflate'), + 'br': ('html-br.bin', 'br') } + class HttpCompressionTest(TestCase): def setUp(self): @@ -50,7 +53,8 @@ class HttpCompressionTest(TestCase): request = Request('http://scrapytest.org') assert 'Accept-Encoding' not in request.headers self.mw.process_request(request, self.spider) - self.assertEqual(request.headers.get('Accept-Encoding'), b'gzip,deflate') + self.assertEqual(request.headers.get('Accept-Encoding'), + b','.join(ACCEPTED_ENCODINGS)) def test_process_response_gzip(self): response = self._getresponse('gzip') @@ -62,6 +66,19 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b'