RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1654 from aron-bordin/show_maxdownsize_warn_once 

[MRG+1] show download warnsize once
This commit is contained in:
Nicolás Alejandro Ramírez Quiros 2016-01-19 11:30:54 -03:00
parent 49fe631d89 f57121c77b
commit 6827eab2c5
1 changed files with 7 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scrapy/core/downloader/handlers/http11.py
View File

@ -295,6 +295,7 @@ class _ResponseReader(protocol.Protocol):
self._bodybuf = BytesIO()
self._maxsize = maxsize
self._warnsize = warnsize
self._reached_warnsize = False
self._bytes_received = 0
def dataReceived(self, bodyBytes):
@ -308,11 +309,12 @@ class _ResponseReader(protocol.Protocol):
'maxsize': self._maxsize})
self._finished.cancel()
if self._warnsize and self._bytes_received > self._warnsize:
logger.warning("Received (%(bytes)s) bytes larger than download "
"warn size (%(warnsize)s).",
{'bytes': self._bytes_received,
'warnsize': self._warnsize})
if self._warnsize and self._bytes_received > self._warnsize and not self._reached_warnsize:
self._reached_warnsize = True
logger.warning("Received more bytes than download "
"warn size (%(warnsize)s) in request %(request)s.",
{'warnsize': self._warnsize,
'request': self._request})
def connectionLost(self, reason):
if self._finished.called: