diff --git a/requirements.txt b/requirements.txt index 362d05013..f92603d3d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,4 +8,3 @@ six>=1.5.2 PyDispatcher>=2.0.5 service_identity parsel>=1.1 -brotlipy==0.6 diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 9202fd8da..04c9e355d 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,6 +1,5 @@ import zlib -import brotli from scrapy.utils.gz import gunzip, is_gzipped from scrapy.http import Response, TextResponse @@ -8,10 +7,18 @@ from scrapy.responsetypes import responsetypes from scrapy.exceptions import NotConfigured +ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] + +try: + import brotli + ACCEPTED_ENCODINGS.append(b'br') +except ImportError: + pass + + class HttpCompressionMiddleware(object): """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): @@ -19,7 +26,8 @@ class HttpCompressionMiddleware(object): return cls() def process_request(self, request, spider): - request.headers.setdefault('Accept-Encoding', 'gzip,deflate,br') + request.headers.setdefault('Accept-Encoding', + ",".join(ACCEPTED_ENCODINGS)) def process_response(self, request, response, spider): @@ -57,7 +65,6 @@ class HttpCompressionMiddleware(object): # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx # http://www.gzip.org/zlib/zlib_faq.html#faq38 body = zlib.decompress(body, -15) - if encoding == b"br": + if encoding == b'br' and b'br' in ACCEPTED_ENCODINGS: body = brotli.decompress(body) return body - diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index c47de54ed..b47b267e2 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,6 +1,6 @@ from io import BytesIO from unittest import TestCase -from os.path import join, abspath, dirname +from os.path import join from gzip import GzipFile from scrapy.spiders import Spider @@ -20,6 +20,7 @@ FORMAT = { 'br': ('html-br.bin', 'br') } + class HttpCompressionTest(TestCase): def setUp(self):