From 016875fd513c66b7ee1bcc80a1f2ae0d6ffda2d2 Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Thu, 3 Dec 2015 15:30:06 +0300 Subject: [PATCH 1/2] added more verbosity for log and for exception when download is cancelled because of a size limit --- scrapy/core/downloader/handlers/http11.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 31412a0f4..78f3f74fa 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -239,11 +239,16 @@ class ScrapyAgent(object): expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 if maxsize and expected_size > maxsize: - logger.error("Expected response size (%(size)s) larger than " - "download max size (%(maxsize)s).", - {'size': expected_size, 'maxsize': maxsize}) + error_message = ( + "Cancelling download of {url}: expected response " + "size ({size}) larger than " + "download max size ({maxsize}).".format( + url=request.url, size=expected_size, maxsize=maxsize + ) + ) + logger.error(error_message) txresponse._transport._producer.loseConnection() - raise defer.CancelledError() + raise defer.CancelledError(error_message) if warnsize and expected_size > warnsize: logger.warning("Expected response size (%(size)s) larger than " From 37289815ac98404458c43159abfd46dd54ed4b7b Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Thu, 3 Dec 2015 16:15:50 +0300 Subject: [PATCH 2/2] code formatting fix --- scrapy/core/downloader/handlers/http11.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 78f3f74fa..9d075e877 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -239,13 +239,11 @@ class ScrapyAgent(object): expected_size = txresponse.length if txresponse.length != UNKNOWN_LENGTH else -1 if maxsize and expected_size > maxsize: - error_message = ( - "Cancelling download of {url}: expected response " - "size ({size}) larger than " - "download max size ({maxsize}).".format( - url=request.url, size=expected_size, maxsize=maxsize - ) - ) + error_message = ("Cancelling download of {url}: expected response " + "size ({size}) larger than " + "download max size ({maxsize})." + ).format(url=request.url, size=expected_size, maxsize=maxsize) + logger.error(error_message) txresponse._transport._producer.loseConnection() raise defer.CancelledError(error_message)