diff --git a/scrapy/tests/sample_data/compressed/truncated-crc-error-short.gz b/scrapy/tests/sample_data/compressed/truncated-crc-error-short.gz new file mode 100644 index 000000000..1e0f9ddd0 Binary files /dev/null and b/scrapy/tests/sample_data/compressed/truncated-crc-error-short.gz differ diff --git a/scrapy/tests/test_utils_gz.py b/scrapy/tests/test_utils_gz.py index eb2acdd13..c7107f6a8 100644 --- a/scrapy/tests/test_utils_gz.py +++ b/scrapy/tests/test_utils_gz.py @@ -22,3 +22,7 @@ class GzTest(unittest.TestCase): with open(join(SAMPLEDIR, 'feed-sample1.xml'), 'rb') as f: self.assertRaises(IOError, gunzip, f.read()) + def test_gunzip_truncated_short(self): + with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f: + text = gunzip(f.read()) + assert text.endswith('') diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py index 57bcb2d60..7fe82ed39 100644 --- a/scrapy/utils/gz.py +++ b/scrapy/utils/gz.py @@ -17,7 +17,9 @@ def gunzip(data): except (IOError, struct.error): # complete only if there is some data, otherwise re-raise # see issue 87 about catching struct.error - if output: + # some pages are quite small so output is '' and f.extrabuf + # contains the whole page content + if output or f.extrabuf: output += f.extrabuf break else: