diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index f504302e2..4e7feeeaf 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,10 +1,12 @@ import io +import warnings import zlib -from scrapy.utils.gz import gunzip +from scrapy.exceptions import NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.exceptions import NotConfigured +from scrapy.utils.deprecate import ScrapyDeprecationWarning +from scrapy.utils.gz import gunzip ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] @@ -25,11 +27,25 @@ except ImportError: class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" + def __init__(self, stats=None): + self.stats = stats + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): raise NotConfigured - return cls() + try: + return cls(stats=crawler.stats) + except TypeError: + warnings.warn( + "HttpCompressionMiddleware subclasses must either modify " + "their '__init__' method to support a 'stats' parameter or " + "reimplement the 'from_crawler' method.", + ScrapyDeprecationWarning, + ) + result = cls() + result.stats = crawler.stats + return result def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', @@ -44,6 +60,9 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) + if self.stats: + self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) + self.stats.inc_value('httpcompression/response_count', spider=spider) respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 4c5bfc577..40e9f3a96 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,13 +1,16 @@ -from io import BytesIO -from unittest import TestCase, SkipTest -from os.path import join from gzip import GzipFile +from io import BytesIO +from os.path import join +from unittest import TestCase, SkipTest +from warnings import catch_warnings from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip +from scrapy.utils.test import get_crawler from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -32,8 +35,10 @@ FORMAT = { class HttpCompressionTest(TestCase): def setUp(self): - self.spider = Spider('foo') - self.mw = HttpCompressionMiddleware() + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('scrapytest.org') + self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) + self.crawler.stats.open_spider(self.spider) def _getresponse(self, coding): if coding not in FORMAT: @@ -56,6 +61,34 @@ class HttpCompressionTest(TestCase): response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'}) return response + def assertStatsEqual(self, key, value): + self.assertEqual( + self.crawler.stats.get_value(key, spider=self.spider), + value, + str(self.crawler.stats.get_stats(self.spider)) + ) + + def test_setting_false_compression_enabled(self): + self.assertRaises( + NotConfigured, + HttpCompressionMiddleware.from_crawler, + get_crawler(settings_dict={'COMPRESSION_ENABLED': False}) + ) + + def test_setting_default_compression_enabled(self): + self.assertIsInstance( + HttpCompressionMiddleware.from_crawler(get_crawler()), + HttpCompressionMiddleware + ) + + def test_setting_true_compression_enabled(self): + self.assertIsInstance( + HttpCompressionMiddleware.from_crawler( + get_crawler(settings_dict={'COMPRESSION_ENABLED': True}) + ), + HttpCompressionMiddleware + ) + def test_process_request(self): request = Request('http://scrapytest.org') assert 'Accept-Encoding' not in request.headers @@ -72,6 +105,20 @@ class HttpCompressionTest(TestCase): assert newresponse is not response assert newresponse.body.startswith(b'