diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 04c9e355d..2fc1bb8eb 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -27,7 +27,7 @@ class HttpCompressionMiddleware(object): def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', - ",".join(ACCEPTED_ENCODINGS)) + b",".join(ACCEPTED_ENCODINGS)) def process_response(self, request, response, spider): diff --git a/tests/requirements.txt b/tests/requirements.txt index 9d0c3c996..9baa4be21 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,6 +6,7 @@ pytest==2.9.2 pytest-twisted pytest-cov==2.2.1 jmespath +brotlipy==0.6 testfixtures # optional for shell wrapper tests bpython diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index b47b267e2..7924fb3b5 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,11 +1,12 @@ from io import BytesIO -from unittest import TestCase +from unittest import TestCase, SkipTest from os.path import join from gzip import GzipFile from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse -from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware +from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, \ + ACCEPTED_ENCODINGS from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -53,7 +54,7 @@ class HttpCompressionTest(TestCase): assert 'Accept-Encoding' not in request.headers self.mw.process_request(request, self.spider) self.assertEqual(request.headers.get('Accept-Encoding'), - b'gzip,deflate,br') + b','.join(ACCEPTED_ENCODINGS)) def test_process_response_gzip(self): response = self._getresponse('gzip') @@ -66,6 +67,10 @@ class HttpCompressionTest(TestCase): assert 'Content-Encoding' not in newresponse.headers def test_process_response_br(self): + try: + import brotli + except ImportError: + raise SkipTest("no brotli") response = self._getresponse('br') request = response.request self.assertEqual(response.headers['Content-Encoding'], b'br')