From 70c82d33c00538228314dd6cef0253b70f8627e8 Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Sun, 20 Sep 2020 16:24:05 +0300 Subject: [PATCH 1/9] httpcompression stats added (#4797) --- scrapy/downloadermiddlewares/httpcompression.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 727c41466..b34f76f21 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -37,6 +37,8 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) + spider.crawler.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) + spider.crawler.stats.inc_value('httpcompression/response_count', spider=spider) respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) From c22e810658b227095ea516ed61e51e6be41068ce Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Tue, 22 Sep 2020 07:47:37 +0300 Subject: [PATCH 2/9] httocompression tests added --- .../downloadermiddlewares/httpcompression.py | 9 ++++--- ...st_downloadermiddleware_httpcompression.py | 27 +++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index b34f76f21..ca80e9444 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -18,11 +18,14 @@ except ImportError: class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" + def __init__(self, stats): + self.stats = stats + @classmethod def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): raise NotConfigured - return cls() + return cls(crawler.stats) def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', @@ -37,8 +40,8 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) - spider.crawler.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) - spider.crawler.stats.inc_value('httpcompression/response_count', spider=spider) + self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) + self.stats.inc_value('httpcompression/response_count', spider=spider) respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index a806f55ce..998749c2e 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -8,6 +8,7 @@ from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip +from scrapy.utils.test import get_crawler from tests import tests_datadir from w3lib.encoding import resolve_encoding @@ -26,8 +27,10 @@ FORMAT = { class HttpCompressionTest(TestCase): def setUp(self): - self.spider = Spider('foo') - self.mw = HttpCompressionMiddleware() + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('scrapytest.org') + self.mw = HttpCompressionMiddleware(self.crawler.stats) + self.crawler.stats.open_spider(self.spider) def _getresponse(self, coding): if coding not in FORMAT: @@ -50,6 +53,13 @@ class HttpCompressionTest(TestCase): response.request = Request('http://scrapytest.org', headers={'Accept-Encoding': 'gzip, deflate'}) return response + def assertStatsEqual(self, key, value): + self.assertEqual( + self.crawler.stats.get_value(key, spider=self.spider), + value, + str(self.crawler.stats.get_stats(self.spider)) + ) + def test_process_request(self): request = Request('http://scrapytest.org') assert 'Accept-Encoding' not in request.headers @@ -66,6 +76,7 @@ class HttpCompressionTest(TestCase): assert newresponse is not response assert newresponse.body.startswith(b' Date: Tue, 6 Oct 2020 18:59:57 +0300 Subject: [PATCH 3/9] httocompression/response_bytes tests added --- tests/test_downloadermiddleware_httpcompression.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 998749c2e..144fd3378 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -77,6 +77,7 @@ class HttpCompressionTest(TestCase): assert newresponse.body.startswith(b' Date: Tue, 13 Oct 2020 18:35:06 +0300 Subject: [PATCH 4/9] __init__ stats parameter - optional, stats==None - covered. --- scrapy/downloadermiddlewares/httpcompression.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index ca80e9444..8980e9ca4 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -18,7 +18,7 @@ except ImportError: class HttpCompressionMiddleware: """This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites""" - def __init__(self, stats): + def __init__(self, stats=None): self.stats = stats @classmethod @@ -40,8 +40,9 @@ class HttpCompressionMiddleware: if content_encoding: encoding = content_encoding.pop() decoded_body = self._decode(response.body, encoding.lower()) - self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) - self.stats.inc_value('httpcompression/response_count', spider=spider) + if self.stats: + self.stats.inc_value('httpcompression/response_bytes', len(decoded_body), spider=spider) + self.stats.inc_value('httpcompression/response_count', spider=spider) respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) From d32d0d27393ce55490c44d9fb039130320461865 Mon Sep 17 00:00:00 2001 From: GeorgeA92 Date: Tue, 13 Oct 2020 18:36:41 +0300 Subject: [PATCH 5/9] testcase added for HttpCompressionMiddleware with no stats --- tests/test_downloadermiddleware_httpcompression.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 144fd3378..5f8e76e0a 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -29,7 +29,7 @@ class HttpCompressionTest(TestCase): def setUp(self): self.crawler = get_crawler(Spider) self.spider = self.crawler._create_spider('scrapytest.org') - self.mw = HttpCompressionMiddleware(self.crawler.stats) + self.mw = HttpCompressionMiddleware.from_crawler(self.crawler) self.crawler.stats.open_spider(self.spider) def _getresponse(self, coding): @@ -79,6 +79,18 @@ class HttpCompressionTest(TestCase): self.assertStatsEqual('httpcompression/response_count', 1) self.assertStatsEqual('httpcompression/response_bytes', 74837) + def test_process_response_gzip_no_stats(self): + mw = HttpCompressionMiddleware() + response = self._getresponse('gzip') + request = response.request + + self.assertEqual(response.headers['Content-Encoding'], b'gzip') + newresponse = mw.process_response(request, response, self.spider) + self.assertEqual(mw.stats, None) + assert newresponse is not response + assert newresponse.body.startswith(b' Date: Tue, 13 Oct 2020 18:41:58 +0300 Subject: [PATCH 6/9] testcase added for COMPRESSION_ENABLED setting --- ...st_downloadermiddleware_httpcompression.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 5f8e76e0a..c2bcbd63c 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -6,6 +6,7 @@ from gzip import GzipFile from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS +from scrapy.exceptions import NotConfigured from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler @@ -60,6 +61,27 @@ class HttpCompressionTest(TestCase): str(self.crawler.stats.get_stats(self.spider)) ) + def test_setting_false_compression_enabled(self): + self.assertRaises( + NotConfigured, + HttpCompressionMiddleware.from_crawler, + get_crawler(settings_dict={'COMPRESSION_ENABLED': False}) + ) + + def test_setting_default_compression_enabled(self): + self.assertIsInstance( + HttpCompressionMiddleware.from_crawler(get_crawler()), + HttpCompressionMiddleware + ) + + def test_setting_true_compression_enabled(self): + self.assertIsInstance( + HttpCompressionMiddleware.from_crawler( + get_crawler(settings_dict={'COMPRESSION_ENABLED': True}) + ), + HttpCompressionMiddleware + ) + def test_process_request(self): request = Request('http://scrapytest.org') assert 'Accept-Encoding' not in request.headers From e9c3188189cffc965797b1b77fc5dc5cfa06b5cb Mon Sep 17 00:00:00 2001 From: Georgiy Zatserklianyi Date: Fri, 30 Oct 2020 21:23:29 +0200 Subject: [PATCH 7/9] Update scrapy/downloadermiddlewares/httpcompression.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adrián Chaves --- scrapy/downloadermiddlewares/httpcompression.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 8980e9ca4..87f744956 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -25,7 +25,12 @@ class HttpCompressionMiddleware: def from_crawler(cls, crawler): if not crawler.settings.getbool('COMPRESSION_ENABLED'): raise NotConfigured - return cls(crawler.stats) + try: + return cls(stats=crawler.stats) + except TypeError: + result = cls() + result.stats = crawler.stats + return result def process_request(self, request, spider): request.headers.setdefault('Accept-Encoding', From a3e53027ec35498dcb931404e02689877da6aeb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 14:16:26 +0100 Subject: [PATCH 8/9] Test HttpCompressionMiddleware subclasses with custom, parameterless __init__ --- .../downloadermiddlewares/httpcompression.py | 12 +++++-- ...st_downloadermiddleware_httpcompression.py | 36 ++++++++++++++++--- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 87f744956..1808154d2 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -1,9 +1,11 @@ +import warnings import zlib -from scrapy.utils.gz import gunzip +from scrapy.exceptions import NotConfigured from scrapy.http import Response, TextResponse from scrapy.responsetypes import responsetypes -from scrapy.exceptions import NotConfigured +from scrapy.utils.deprecate import ScrapyDeprecationWarning +from scrapy.utils.gz import gunzip ACCEPTED_ENCODINGS = [b'gzip', b'deflate'] @@ -28,6 +30,12 @@ class HttpCompressionMiddleware: try: return cls(stats=crawler.stats) except TypeError: + warnings.warn( + "HttpCompressionMiddleware subclasses must either modify " + "their '__init__' method to support a 'stats' parameter or " + "reimplement the 'from_crawler' method.", + ScrapyDeprecationWarning, + ) result = cls() result.stats = crawler.stats return result diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index c2bcbd63c..2ee87aa67 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -1,12 +1,13 @@ -from io import BytesIO -from unittest import TestCase, SkipTest -from os.path import join from gzip import GzipFile +from io import BytesIO +from os.path import join +from unittest import TestCase, SkipTest +from warnings import catch_warnings from scrapy.spiders import Spider from scrapy.http import Response, Request, HtmlResponse from scrapy.downloadermiddlewares.httpcompression import HttpCompressionMiddleware, ACCEPTED_ENCODINGS -from scrapy.exceptions import NotConfigured +from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.responsetypes import responsetypes from scrapy.utils.gz import gunzip from scrapy.utils.test import get_crawler @@ -321,3 +322,30 @@ class HttpCompressionTest(TestCase): self.assertEqual(response.body, b'') self.assertStatsEqual('httpcompression/response_count', None) self.assertStatsEqual('httpcompression/response_bytes', None) + + +class HttpCompressionSubclassTest(TestCase): + + def test_init_missing_stats(self): + class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware): + + def __init__(self): + super().__init__() + + crawler = get_crawler(Spider) + with catch_warnings(record=True) as caught_warnings: + instance = HttpCompressionMiddlewareSubclass.from_crawler(crawler) + messages = tuple( + str(warning.message) for warning in caught_warnings + if warning.category is ScrapyDeprecationWarning + ) + self.assertEqual( + messages, + ( + ( + "HttpCompressionMiddleware subclasses must either modify " + "their '__init__' method to support a 'stats' parameter " + "or reimplement the 'from_crawler' method." + ), + ) + ) From ee98771fa72aa5e109292af5d734e5c49bae64e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Fri, 6 Nov 2020 16:42:32 +0100 Subject: [PATCH 9/9] Remove unused variable --- tests/test_downloadermiddleware_httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py index 2ee87aa67..38d8534ca 100644 --- a/tests/test_downloadermiddleware_httpcompression.py +++ b/tests/test_downloadermiddleware_httpcompression.py @@ -334,7 +334,7 @@ class HttpCompressionSubclassTest(TestCase): crawler = get_crawler(Spider) with catch_warnings(record=True) as caught_warnings: - instance = HttpCompressionMiddlewareSubclass.from_crawler(crawler) + HttpCompressionMiddlewareSubclass.from_crawler(crawler) messages = tuple( str(warning.message) for warning in caught_warnings if warning.category is ScrapyDeprecationWarning