From 03cf5f1bd2019127933ea4a6358a7d47743efcf6 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 17:18:57 +0100 Subject: [PATCH 1/4] Remove ChunkedTransferMiddleware from default settings --- docs/topics/downloader-middleware.rst | 8 +++++++- docs/topics/settings.rst | 1 - scrapy/settings/default_settings.py | 1 - 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 15069e56e..dca5ec6a0 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -665,7 +665,13 @@ ChunkedTransferMiddleware .. class:: ChunkedTransferMiddleware - This middleware adds support for `chunked transfer encoding`_ + This middleware adds support for `chunked transfer encoding`_. + +.. note:: + This middleware is not enabled nor used by Scrapy downloader anymore. + In fact, Scrapy downloader has built-in support for chunked transfers, + so this middleware has no effect in practice. + HttpProxyMiddleware ------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8540308fe..2195e4233 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -468,7 +468,6 @@ Default:: 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600, 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750, - 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830, 'scrapy.downloadermiddlewares.stats.DownloaderStats': 850, 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900, } diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 18d5ebbbb..61f4bd567 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -102,7 +102,6 @@ DOWNLOADER_MIDDLEWARES_BASE = { 'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600, 'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700, 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750, - 'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830, 'scrapy.downloadermiddlewares.stats.DownloaderStats': 850, 'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900, # Downloader side From e6f174b01535810fffeae2d020bc66a6a25ba4e8 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 17:33:27 +0100 Subject: [PATCH 2/4] Add deprecation warning for ChunkedTransfer middleware --- scrapy/downloadermiddlewares/chunked.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py index 57e97e4d2..fd90aab2a 100644 --- a/scrapy/downloadermiddlewares/chunked.py +++ b/scrapy/downloadermiddlewares/chunked.py @@ -1,6 +1,14 @@ +import warnings + +from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.http import decode_chunked_transfer +warnings.warn("Module `scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware` " + "is deprecated, chunked transfers are supported by default.", + ScrapyDeprecationWarning, stacklevel=2) + + class ChunkedTransferMiddleware(object): """This middleware adds support for chunked transfer encoding, as documented in: http://en.wikipedia.org/wiki/Chunked_transfer_encoding From 8cffb4bbefcebd10ec12ee2678fd490edf149576 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 17:50:21 +0100 Subject: [PATCH 3/4] Update warning wording --- scrapy/downloadermiddlewares/chunked.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/chunked.py b/scrapy/downloadermiddlewares/chunked.py index fd90aab2a..64d94c489 100644 --- a/scrapy/downloadermiddlewares/chunked.py +++ b/scrapy/downloadermiddlewares/chunked.py @@ -4,8 +4,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.http import decode_chunked_transfer -warnings.warn("Module `scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware` " - "is deprecated, chunked transfers are supported by default.", +warnings.warn("Module `scrapy.downloadermiddlewares.chunked` is deprecated, " + "chunked transfers are supported by default.", ScrapyDeprecationWarning, stacklevel=2) From 059085b5b4a54901c4fd0730e43b156a6d8eeb51 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 24 Nov 2016 18:23:34 +0100 Subject: [PATCH 4/4] Remove docs for deprecated ChunkedTransfer middleware --- docs/topics/downloader-middleware.rst | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index dca5ec6a0..29d9b0298 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -657,22 +657,6 @@ Default: ``True`` Whether the Compression middleware will be enabled. -ChunkedTransferMiddleware -------------------------- - -.. module:: scrapy.downloadermiddlewares.chunked - :synopsis: Chunked Transfer Middleware - -.. class:: ChunkedTransferMiddleware - - This middleware adds support for `chunked transfer encoding`_. - -.. note:: - This middleware is not enabled nor used by Scrapy downloader anymore. - In fact, Scrapy downloader has built-in support for chunked transfers, - so this middleware has no effect in practice. - - HttpProxyMiddleware ------------------- @@ -976,4 +960,3 @@ The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. .. _DBM: https://en.wikipedia.org/wiki/Dbm .. _anydbm: https://docs.python.org/2/library/anydbm.html -.. _chunked transfer encoding: https://en.wikipedia.org/wiki/Chunked_transfer_encoding