httpcompression middleware improvement

scrapy/tests/test_utils_gz.py

@@ -22,3 +22,7 @@ class GzTest(unittest.TestCase):
         with open(join(SAMPLEDIR, 'feed-sample1.xml'), 'rb') as f:
             self.assertRaises(IOError, gunzip, f.read())
 
+    def test_gunzip_truncated_short(self):
+        with open(join(SAMPLEDIR, 'truncated-crc-error-short.gz'), 'rb') as f:
+            text = gunzip(f.read())
+            assert text.endswith('</html>')

scrapy/utils/gz.py

@@ -17,7 +17,9 @@ def gunzip(data):
         except (IOError, struct.error):
             # complete only if there is some data, otherwise re-raise
             # see issue 87 about catching struct.error
-            if output:
+            # some pages are quite small so output is '' and f.extrabuf
+            # contains the whole page content
+            if output or f.extrabuf:
                 output += f.extrabuf
                 break
             else: