diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 8400fc057..04bb5a696 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -38,6 +38,7 @@ COMMANDS_MODULE = "" COMPRESSION_ENABLED = True COMPRESSION_KEEP_ENCODING_HEADER = False + CONCURRENT_ITEMS = 100 CONCURRENT_REQUESTS = 16 diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index 9d337c530..856b9240b 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -86,6 +86,8 @@ ROBOTSTXT_OBEY = True #HTTPCACHE_DIR = "httpcache" #HTTPCACHE_IGNORE_HTTP_CODES = [] #HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage" + +# Keep original Content-Encoding header COMPRESSION_KEEP_ENCODING_HEADER = True # Set settings whose default value is deprecated to a future-proof value diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index fac5588ff..f9e9b4661 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -13,6 +13,7 @@ from scrapy.downloadermiddlewares.httpcompression import ( from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import HtmlResponse, Request, Response from scrapy.responsetypes import responsetypes +from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler @@ -109,23 +110,25 @@ class HttpCompressionTest(TestCase): self.assertEqual(response.headers["Content-Encoding"], b"gzip") newresponse = self.mw.process_response(request, response, self.spider) - assert newresponse is not response - assert newresponse.body.startswith(b"