added new downloader middleware: ChunkedTransferMiddleware

docs/topics/downloader-middleware.rst

@@ -434,6 +434,16 @@ HttpCompressionMiddleware
    This middleware allows compressed (gzip, deflate) traffic to be
    sent/received from web sites.
 
+ChunkedTransferMiddleware
+-------------------------
+
+.. module:: scrapy.contrib.downloadermiddleware.chunked
+   :synopsis: Chunked Transfer Middleware
+
+.. class:: ChunkedTransferMiddleware 
+
+   This middleware adds support for `chunked transfer encoding`_
+
 HttpProxyMiddleware
 -------------------
 
@@ -637,3 +647,4 @@ UserAgentMiddleware
 
 .. _DBM: http://en.wikipedia.org/wiki/Dbm
 .. _anydbm: http://docs.python.org/library/anydbm.html
+.. _chunked transfer encoding: http://en.wikipedia.org/wiki/Chunked_transfer_encoding

docs/topics/settings.rst

@@ -412,6 +412,7 @@ Default::
         'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
         'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750,
         'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800,
+        'scrapy.contrib.downloadermiddleware.chunked.ChunkedTransferMiddleware': 830,
         'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850,
         'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900,
     }

scrapy/contrib/downloadermiddleware/chunked.py

@@ -0,0 +1,13 @@
+from scrapy.utils.http import decode_chunked_transfer
+
+
+class ChunkedTransferMiddleware(object):
+    """This middleware adds support for chunked transfer encoding, as
+    documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding
+    """
+
+    def process_response(self, request, response, spider):
+        if response.headers.get('Transfer-Encoding') == 'chunked':
+            body = decode_chunked_transfer(response.body)
+            return response.replace(body=body)
+        return response

scrapy/settings/default_settings.py

@@ -81,6 +81,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
     'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700,
     'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750,
     'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800,
+    'scrapy.contrib.downloadermiddleware.chunked.ChunkedTransferMiddleware': 830,
     'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850,
     'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900,
     # Downloader side

scrapy/tests/test_utils_http.py

@@ -0,0 +1,20 @@
+import unittest
+
+from scrapy.utils.http import decode_chunked_transfer
+
+class ChunkedTest(unittest.TestCase):
+
+    def test_decode_chunked_transfer(self):
+        """Example taken from: http://en.wikipedia.org/wiki/Chunked_transfer_encoding"""
+        chunked_body = "25\r\n" + "This is the data in the first chunk\r\n\r\n"
+        chunked_body += "1C\r\n" + "and this is the second one\r\n\r\n"
+        chunked_body += "3\r\n" + "con\r\n"
+        chunked_body += "8\r\n" + "sequence\r\n"
+        chunked_body += "0\r\n\r\n"
+        body = decode_chunked_transfer(chunked_body)
+        self.assertEqual(body, \
+            "This is the data in the first chunk\r\n" +
+            "and this is the second one\r\n" +
+            "consequence")
+
+

scrapy/utils/http.py

@@ -5,3 +5,22 @@ For new code, always import from w3lib.http instead of this module
 """
 
 from w3lib.http import *
+
+def decode_chunked_transfer(chunked_body):
+    """Parsed body received with chunked transfer encoding, and return the
+    decoded body.
+
+    For more info see:
+    http://en.wikipedia.org/wiki/Chunked_transfer_encoding
+
+    """
+    body, h, t = '', '', chunked_body
+    while t:
+        h, t = t.split('\r\n', 1)
+        if h == '0':
+            break
+        size = int(h, 16)
+        body += t[:size]
+        t = t[size+2:]
+    return body
+