diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index f7aee0a97..d845d671c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -434,6 +434,16 @@ HttpCompressionMiddleware This middleware allows compressed (gzip, deflate) traffic to be sent/received from web sites. +ChunkedTransferMiddleware +------------------------- + +.. module:: scrapy.contrib.downloadermiddleware.chunked + :synopsis: Chunked Transfer Middleware + +.. class:: ChunkedTransferMiddleware + + This middleware adds support for `chunked transfer encoding`_ + HttpProxyMiddleware ------------------- @@ -637,3 +647,4 @@ UserAgentMiddleware .. _DBM: http://en.wikipedia.org/wiki/Dbm .. _anydbm: http://docs.python.org/library/anydbm.html +.. _chunked transfer encoding: http://en.wikipedia.org/wiki/Chunked_transfer_encoding diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 24da5c2e6..8f4b194ce 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -412,6 +412,7 @@ Default:: 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700, 'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750, 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800, + 'scrapy.contrib.downloadermiddleware.chunked.ChunkedTransferMiddleware': 830, 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850, 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, } diff --git a/scrapy/contrib/downloadermiddleware/chunked.py b/scrapy/contrib/downloadermiddleware/chunked.py new file mode 100644 index 000000000..57e97e4d2 --- /dev/null +++ b/scrapy/contrib/downloadermiddleware/chunked.py @@ -0,0 +1,13 @@ +from scrapy.utils.http import decode_chunked_transfer + + +class ChunkedTransferMiddleware(object): + """This middleware adds support for chunked transfer encoding, as + documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding + """ + + def process_response(self, request, response, spider): + if response.headers.get('Transfer-Encoding') == 'chunked': + body = decode_chunked_transfer(response.body) + return response.replace(body=body) + return response diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 9f38c2010..2c7a5e8a0 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -81,6 +81,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.contrib.downloadermiddleware.cookies.CookiesMiddleware': 700, 'scrapy.contrib.downloadermiddleware.httpproxy.HttpProxyMiddleware': 750, 'scrapy.contrib.downloadermiddleware.httpcompression.HttpCompressionMiddleware': 800, + 'scrapy.contrib.downloadermiddleware.chunked.ChunkedTransferMiddleware': 830, 'scrapy.contrib.downloadermiddleware.stats.DownloaderStats': 850, 'scrapy.contrib.downloadermiddleware.httpcache.HttpCacheMiddleware': 900, # Downloader side diff --git a/scrapy/tests/test_utils_http.py b/scrapy/tests/test_utils_http.py new file mode 100644 index 000000000..583105673 --- /dev/null +++ b/scrapy/tests/test_utils_http.py @@ -0,0 +1,20 @@ +import unittest + +from scrapy.utils.http import decode_chunked_transfer + +class ChunkedTest(unittest.TestCase): + + def test_decode_chunked_transfer(self): + """Example taken from: http://en.wikipedia.org/wiki/Chunked_transfer_encoding""" + chunked_body = "25\r\n" + "This is the data in the first chunk\r\n\r\n" + chunked_body += "1C\r\n" + "and this is the second one\r\n\r\n" + chunked_body += "3\r\n" + "con\r\n" + chunked_body += "8\r\n" + "sequence\r\n" + chunked_body += "0\r\n\r\n" + body = decode_chunked_transfer(chunked_body) + self.assertEqual(body, \ + "This is the data in the first chunk\r\n" + + "and this is the second one\r\n" + + "consequence") + + diff --git a/scrapy/utils/http.py b/scrapy/utils/http.py index 574ca4744..8b659a22a 100644 --- a/scrapy/utils/http.py +++ b/scrapy/utils/http.py @@ -5,3 +5,22 @@ For new code, always import from w3lib.http instead of this module """ from w3lib.http import * + +def decode_chunked_transfer(chunked_body): + """Parsed body received with chunked transfer encoding, and return the + decoded body. + + For more info see: + http://en.wikipedia.org/wiki/Chunked_transfer_encoding + + """ + body, h, t = '', '', chunked_body + while t: + h, t = t.split('\r\n', 1) + if h == '0': + break + size = int(h, 16) + body += t[:size] + t = t[size+2:] + return body +