RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added test case and code review suggestions

This commit is contained in:
Hassaan Naushahi 2023-09-08 17:30:03 +05:00
parent 918c8ffd86
commit 72962d6f5b
3 changed files with 139 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
scrapy/settings/default_settings.py
View File

@ -38,6 +38,7 @@ COMMANDS_MODULE = ""
COMPRESSION_ENABLED = True
COMPRESSION_KEEP_ENCODING_HEADER = False
CONCURRENT_ITEMS = 100
CONCURRENT_REQUESTS = 16

2
scrapy/templates/project/module/settings.py.tmpl
View File

@ -86,6 +86,8 @@ ROBOTSTXT_OBEY = True
#HTTPCACHE_DIR = "httpcache"
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = "scrapy.extensions.httpcache.FilesystemCacheStorage"
# Keep original Content-Encoding header
COMPRESSION_KEEP_ENCODING_HEADER = True
# Set settings whose default value is deprecated to a future-proof value

173
tests/test_downloadermiddleware_httpcompression.py
View File

@ -13,6 +13,7 @@ from scrapy.downloadermiddlewares.httpcompression import (
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Request, Response
from scrapy.responsetypes import responsetypes
from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils.gz import gunzip
from scrapy.utils.test import get_crawler
@ -109,23 +110,25 @@ class HttpCompressionTest(TestCase):
self.assertEqual(response.headers["Content-Encoding"], b"gzip")
newresponse = self.mw.process_response(request, response, self.spider)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertNotEqual(newresponse, response)
self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
self.assertIn('Content-Encoding', newresponse.headers)
self.assertNotIn(b'gzip', newresponse.headers['Content-Encoding'])
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
def test_process_response_gzip_no_stats(self):
mw = HttpCompressionMiddleware()
mw = HttpCompressionMiddleware(settings=Settings({'COMPRESSION_KEEP_ENCODING_HEADER': True}))
response = self._getresponse("gzip")
request = response.request
self.assertEqual(response.headers["Content-Encoding"], b"gzip")
newresponse = mw.process_response(request, response, self.spider)
self.assertEqual(mw.stats, None)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertNotEqual(newresponse, response)
self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
self.assertIn('Content-Encoding', newresponse.headers)
self.assertNotIn(b'gzip', newresponse.headers['Content-Encoding'])
def test_process_response_br(self):
try:
@ -136,9 +139,10 @@ class HttpCompressionTest(TestCase):
request = response.request
self.assertEqual(response.headers["Content-Encoding"], b"br")
newresponse = self.mw.process_response(request, response, self.spider)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertNotEqual(newresponse, response)
self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
self.assertIn('Content-Encoding', newresponse.headers)
self.assertEqual(b'br', newresponse.headers['Content-Encoding'])
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
@ -158,10 +162,11 @@ class HttpCompressionTest(TestCase):
if raw_content is None:
raw_content = newresponse.body
else:
assert raw_content == newresponse.body
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertEqual(raw_content, newresponse.body)
self.assertNotEqual(newresponse, response)
self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
self.assertIn('Content-Encoding', newresponse.headers)
self.assertEqual(b'zstd', newresponse.headers['Content-Encoding'])
def test_process_response_rawdeflate(self):
response = self._getresponse("rawdeflate")
@ -169,11 +174,13 @@ class HttpCompressionTest(TestCase):
self.assertEqual(response.headers["Content-Encoding"], b"deflate")
newresponse = self.mw.process_response(request, response, self.spider)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertNotEqual(newresponse, response)
self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
self.assertIn('Content-Encoding', newresponse.headers)
self.assertIn(b'deflate', newresponse.headers['Content-Encoding'])
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74840)
self.assertIn(b'decoded', newresponse.flags)
def test_process_response_zlibdelate(self):
response = self._getresponse("zlibdeflate")
@ -181,20 +188,23 @@ class HttpCompressionTest(TestCase):
self.assertEqual(response.headers["Content-Encoding"], b"deflate")
newresponse = self.mw.process_response(request, response, self.spider)
assert newresponse is not response
assert newresponse.body.startswith(b"<!DOCTYPE")
assert "Content-Encoding" not in newresponse.headers
self.assertNotEqual(newresponse, response)
self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
self.assertIn('Content-Encoding', newresponse.headers)
self.assertIn(b'deflate', newresponse.headers['Content-Encoding'])
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74840)
self.assertIn(b'decoded', newresponse.flags)
def test_process_response_plain(self):
response = Response("http://scrapytest.org", body=b"<!DOCTYPE...")
request = Request("http://scrapytest.org")
assert not response.headers.get("Content-Encoding")
self.assertTrue(not response.headers.get('Content-Encoding'))
newresponse = self.mw.process_response(request, response, self.spider)
assert newresponse is response
assert newresponse.body.startswith(b"<!DOCTYPE")
self.assertEqual(newresponse, response)
self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
self.assertNotIn('Content-Encoding', newresponse.headers)
self.assertStatsEqual("httpcompression/response_count", None)
self.assertStatsEqual("httpcompression/response_bytes", None)
@ -203,8 +213,10 @@ class HttpCompressionTest(TestCase):
response.headers["Content-Encoding"] = ["uuencode", "gzip"]
request = response.request
newresponse = self.mw.process_response(request, response, self.spider)
assert newresponse is not response
self.assertEqual(newresponse.headers.getlist("Content-Encoding"), [b"uuencode"])
self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode'])
self.assertNotEqual(newresponse, response)
self.assertEqual(newresponse.headers.getlist('Content-Encoding'), [b'uuencode', b'gzip'])
self.assertIn(b'decoded', newresponse.flags)
def test_process_response_encoding_inside_body(self):
headers = {
@ -225,11 +237,14 @@ class HttpCompressionTest(TestCase):
request = Request("http://www.example.com/")
newresponse = self.mw.process_response(request, response, self.spider)
assert isinstance(newresponse, HtmlResponse)
self.assertIsInstance(newresponse, HtmlResponse)
self.assertEqual(newresponse.body, plainbody)
self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 104)
self.assertIn('Content-Encoding', newresponse.headers)
self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
self.assertIn(b'decoded', newresponse.flags)
def test_process_response_force_recalculate_encoding(self):
headers = {
@ -250,11 +265,14 @@ class HttpCompressionTest(TestCase):
request = Request("http://www.example.com/")
newresponse = self.mw.process_response(request, response, self.spider)
assert isinstance(newresponse, HtmlResponse)
self.assertIsInstance(newresponse, HtmlResponse)
self.assertEqual(newresponse.body, plainbody)
self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
self.assertIn('Content-Encoding', newresponse.headers)
self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 104)
self.assertIn(b'decoded', newresponse.flags)
def test_process_response_no_content_type_header(self):
headers = {
@ -273,11 +291,14 @@ class HttpCompressionTest(TestCase):
request = Request("http://www.example.com/index")
newresponse = self.mw.process_response(request, response, self.spider)
assert isinstance(newresponse, respcls)
self.assertIsInstance(newresponse, respcls)
self.assertEqual(newresponse.body, plainbody)
self.assertEqual(newresponse.encoding, resolve_encoding("gb2312"))
self.assertIn('Content-Encoding', newresponse.headers)
self.assertEqual(b'identity', newresponse.headers['Content-Encoding'])
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 104)
self.assertIn(b'decode', newresponse.flags)
def test_process_response_gzipped_contenttype(self):
response = self._getresponse("gzip")
@ -287,9 +308,11 @@ class HttpCompressionTest(TestCase):
newresponse = self.mw.process_response(request, response, self.spider)
self.assertIsNot(newresponse, response)
self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
self.assertNotIn("Content-Encoding", newresponse.headers)
self.assertIn('Content-Encoding', newresponse.headers)
self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
self.assertIn(b'decoded', newresponse.flags)
def test_process_response_gzip_app_octetstream_contenttype(self):
response = self._getresponse("gzip")
@ -299,9 +322,11 @@ class HttpCompressionTest(TestCase):
newresponse = self.mw.process_response(request, response, self.spider)
self.assertIsNot(newresponse, response)
self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
self.assertNotIn("Content-Encoding", newresponse.headers)
self.assertIn('Content-Encoding', newresponse.headers)
self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
self.assertIn(b'decoded', newresponse.flags)
def test_process_response_gzip_binary_octetstream_contenttype(self):
response = self._getresponse("x-gzip")
@ -311,9 +336,11 @@ class HttpCompressionTest(TestCase):
newresponse = self.mw.process_response(request, response, self.spider)
self.assertIsNot(newresponse, response)
self.assertTrue(newresponse.body.startswith(b"<!DOCTYPE"))
self.assertNotIn("Content-Encoding", newresponse.headers)
self.assertIn('Content-Encoding', newresponse.headers)
self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 74837)
self.assertIn(b'decoded', newresponse.flags)
def test_process_response_gzipped_gzip_file(self):
"""Test that a gzip Content-Encoded .gz file is gunzipped
@ -360,6 +387,9 @@ class HttpCompressionTest(TestCase):
self.assertEqual(gunzip(newresponse.body), plainbody)
self.assertStatsEqual("httpcompression/response_count", 1)
self.assertStatsEqual("httpcompression/response_bytes", 230)
self.assertIn('Content-Encoding', newresponse.headers)
self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
self.assertIn(b'decoded', newresponse.flags
def test_process_response_head_request_no_decode_required(self):
response = self._getresponse("gzip")
@ -370,12 +400,46 @@ class HttpCompressionTest(TestCase):
newresponse = self.mw.process_response(request, response, self.spider)
self.assertIs(newresponse, response)
self.assertEqual(response.body, b"")
self.assertIn('Content-Encoding', newresponse.headers)
self.assertEqual(b'gzip', newresponse.headers['Content-Encoding'])
self.assertStatsEqual("httpcompression/response_count", None)
self.assertStatsEqual("httpcompression/response_bytes", None)
def test_process_response_keeps_content_encoding_header(self):
settings = {'COMPRESSION_ENABLED': True,
'COMPRESSION_KEEP_ENCODING_HEADER': True}
crawler = get_crawler(Spider, settings)
spider = crawler._create_spider('example.com')
mw = HttpCompressionMiddleware.from_crawler(crawler)
response = self._getresponse('gzip')
request = response.request
self.assertEqual(response.headers['Content-Encoding'], b'gzip')
newresponse = mw.process_response(request, response, spider)
self.assertIsNot(newresponse, response)
self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
self.assertIn('Content-Encoding', newresponse.headers)
self.assertNotIn(b'gzip', newresponse.headers['Content-Encoding'])
self.assertIn(b'decoded', newresponse.flags)
def test_process_response_doesnt_keep_content_encoding_header(self):
settings = {'COMPRESSION_ENABLED': True,
'COMPRESSION_KEEP_ENCODING_HEADER': False}
crawler = get_crawler(Spider, settings)
spider = crawler._create_spider('example.com')
mw = HttpCompressionMiddleware.from_crawler(crawler)
response = self._getresponse('gzip')
request = response.request
self.assertEqual(response.headers['Content-Encoding'], b'gzip')
newresponse = mw.process_response(request, response, spider)
self.assertIsNot(newresponse, response)
self.assertTrue(newresponse.body.startswith(b'<!DOCTYPE'))
self.assertNotIn('Content-Encoding', newresponse.headers)
self.assertIn(b'decoded', newresponse.flags)
class HttpCompressionSubclassTest(TestCase):
def test_init_missing_stats(self):
def test_from_crawler_missing_args(self):
class HttpCompressionMiddlewareSubclass(HttpCompressionMiddleware):
def __init__(self):
super().__init__()
@ -391,10 +455,45 @@ class HttpCompressionSubclassTest(TestCase):
self.assertEqual(
messages,
(
(
"HttpCompressionMiddleware subclasses must either modify "
"their '__init__' method to support a 'stats' parameter "
"or reimplement the 'from_crawler' method."
),
"HttpCompressionMiddleware subclasses must either modify "
"their '__init__' method to support 'stats' and 'settings' parameters "
"or reimplement the 'from_crawler' method.",
"HttpCompressionMiddleware now accepts a 'stats' parameter which should be specified.",
"HttpCompressionMiddleware now accepts a 'settings' parameter which should be specified.",
),
)
def test_init_missing_args(self):
with catch_warnings(record=True) as caught_warnings:
self.assertIsNotNone(HttpCompressionMiddleware(stats=None, settings=None))
messages = tuple(
str(warning.message) for warning in caught_warnings
if warning.category is ScrapyDeprecationWarning
)
self.assertEqual(
messages,
(
"HttpCompressionMiddleware now accepts a 'stats' parameter which should be specified.",
"HttpCompressionMiddleware now accepts a 'settings' parameter which should be specified.",
),
)
def test_init_keep_encoding_header_deprecation_warning(self):
from scrapy.settings import Settings
settings = Settings({'COMPRESSION_KEEP_ENCODING_HEADER': False})
with catch_warnings(record=True) as caught_warnings:
mw = HttpCompressionMiddleware(stats={'foo': 'bar'}, settings=settings)
self.assertIsNotNone(mw)
messages = tuple(
str(warning.message) for warning in caught_warnings
if warning.category is ScrapyDeprecationWarning
)
self.assertEqual(
messages,
(
"Setting COMPRESSION_KEEP_ENCODING_HEADER=False is deprecated",
),
)