diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 64f0eb953..ac4945978 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -533,6 +533,19 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. +HttpCompressionMiddleware Settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: COMPRESSION_ENABLED + +COMPRESSION_ENABLED +^^^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether the Compression middleware will be enabled. + + ChunkedTransferMiddleware ------------------------- diff --git a/scrapy/contrib/downloadermiddleware/httpcompression.py b/scrapy/contrib/downloadermiddleware/httpcompression.py index 25af3e051..60d87da7a 100644 --- a/scrapy/contrib/downloadermiddleware/httpcompression.py +++ b/scrapy/contrib/downloadermiddleware/httpcompression.py @@ -3,12 +3,19 @@ import zlib from scrapy.utils.gz import gunzip from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes +from scrapy.exceptions import NotConfigured class HttpCompressionMiddleware(object): """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - + + @classmethod + def from_crawler(cls, crawler): + if not crawler.settings.getbool('COMPRESSION_ENABLED'): + raise NotConfigured + return cls() + def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', 'x-gzip,gzip,deflate') diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9abbba616..a16110b52 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -26,6 +26,8 @@ CLOSESPIDER_ERRORCOUNT = 0 COMMANDS_MODULE = '' +COMPRESSION_ENABLED = True + CONCURRENT_ITEMS = 100 CONCURRENT_REQUESTS = 16