From f57121c77be82ad7521c3f4ed5cda17ada118ef4 Mon Sep 17 00:00:00 2001 From: Aron Bordin Date: Wed, 30 Dec 2015 13:10:13 -0200 Subject: [PATCH] show download warnsize once --- scrapy/core/downloader/handlers/http11.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 31412a0f4..403accef8 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -292,6 +292,7 @@ class _ResponseReader(protocol.Protocol): self._bodybuf = BytesIO() self._maxsize = maxsize self._warnsize = warnsize + self._reached_warnsize = False self._bytes_received = 0 def dataReceived(self, bodyBytes): @@ -305,11 +306,12 @@ class _ResponseReader(protocol.Protocol): 'maxsize': self._maxsize}) self._finished.cancel() - if self._warnsize and self._bytes_received > self._warnsize: - logger.warning("Received (%(bytes)s) bytes larger than download " - "warn size (%(warnsize)s).", - {'bytes': self._bytes_received, - 'warnsize': self._warnsize}) + if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize: + self._reached_warnsize = True + logger.warning("Received more bytes than download " + "warn size (%(warnsize)s) in request %(request)s.", + {'warnsize': self._warnsize, + 'request': self._request}) def connectionLost(self, reason): if self._finished.called: