fixed bug handling truncated gzipped responses. closes #319

scrapy/contrib/downloadermiddleware/httpcompression.py

@@ -1,7 +1,6 @@
 import zlib
-from gzip import GzipFile
-from cStringIO import StringIO
 
+from scrapy.utils.gz import gunzip
 from scrapy.http import Response, TextResponse
 from scrapy.core.downloader.responsetypes import responsetypes
 
@@ -18,6 +17,8 @@ class HttpCompressionMiddleware(object):
             content_encoding = response.headers.getlist('Content-Encoding')
             if content_encoding:
                 encoding = content_encoding.pop()
+                with open('body', 'w') as f:
+                    f.write(response.body)
                 decoded_body = self._decode(response.body, encoding.lower())
                 respcls = responsetypes.from_args(headers=response.headers, \
                     url=response.url)
@@ -34,7 +35,7 @@ class HttpCompressionMiddleware(object):
 
     def _decode(self, body, encoding):
         if encoding == 'gzip':
-            body = GzipFile(fileobj=StringIO(body)).read()
+            body = gunzip(body)
 
         if encoding == 'deflate':
             try:

scrapy/tests/test_utils_gz.py

@@ -0,0 +1,26 @@
+from __future__ import with_statement
+
+import unittest
+from os.path import join
+
+from scrapy.tests import tests_datadir
+from scrapy.utils.gz import gunzip
+
+SAMPLEDIR = join(tests_datadir, 'compressed')
+
+class GzTest(unittest.TestCase):
+
+    def test_gunzip_basic(self):
+        with open(join(SAMPLEDIR, 'feed-sample1.xml.gz'), 'rb') as f:
+            text = gunzip(f.read())
+            self.assertEqual(len(text), 9950)
+
+    def test_gunzip_truncated(self):
+        with open(join(SAMPLEDIR, 'truncated-crc-error.gz'), 'rb') as f:
+            text = gunzip(f.read())
+            assert text.endswith('</html')
+
+    def test_gunzip_no_gzip_file_raises(self):
+        with open(join(SAMPLEDIR, 'feed-sample1.xml'), 'rb') as f:
+            self.assertRaises(IOError, gunzip, f.read())
+

scrapy/utils/gz.py

@@ -0,0 +1,23 @@
+from cStringIO import StringIO
+from gzip import GzipFile
+
+def gunzip(data):
+    """Gunzip the given data and return as much data as possible.
+
+    This is resilient to CRC checksum errors.
+    """
+    f = GzipFile(fileobj=StringIO(data))
+    output = ''
+    chunk = '.'
+    while chunk:
+        try:
+            chunk = f.read(8196)
+            output += chunk
+        except IOError:
+            # complete only if there is some data, otherwise re-raise
+            if output:
+                output += f.extrabuf
+                break
+            else:
+                raise
+    return output