RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change DOWNLOAD_MAXSIZE logger level from Error to Warning

This commit is contained in:
WinterComes 2019-07-17 22:50:34 +03:00 committed by drs-11
parent f1250177dc
commit 2d8ec9d44f
2 changed files with 12 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
scrapy/core/downloader/handlers/http11.py
View File

@ -394,13 +394,14 @@ class ScrapyAgent:
fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)
if maxsize and expected_size > maxsize:
error_msg = ("Cancelling download of %(url)s: expected response "
"size (%(size)s) larger than download max size (%(maxsize)s).")
error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
warning_msg = ("Expected response size (%(size)s) larger than "
"download max size (%(maxsize)s) in request %(request)s.")
warning_args = {'request': request, 'size': expected_size, 'maxsize': maxsize}
logger.warning(warning_msg, warning_args)
logger.error(error_msg, error_args)
txresponse._transport._producer.loseConnection()
raise defer.CancelledError(error_msg % error_args)
raise defer.CancelledError(warning_msg % warning_args)
if warnsize and expected_size > warnsize:
logger.warning("Expected response size (%(size)s) larger than "
@ -523,11 +524,11 @@ class _ResponseReader(protocol.Protocol):
self._finish_response(flags=["download_stopped"], failure=failure)
if self._maxsize and self._bytes_received > self._maxsize:
logger.error("Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",
{'bytes': self._bytes_received,
'maxsize': self._maxsize,
'request': self._request})
logger.warning("Received (%(bytes)s) bytes larger than download "
"max size (%(maxsize)s) in request %(request)s.",
{'bytes': self._bytes_received,
'maxsize': self._maxsize,
'request': self._request})
# Clear buffer earlier to avoid keeping data in memory for a long time.
self._bodybuf.truncate(0)
self._finished.cancel()

2
tests/test_downloader_handlers.py
View File

@ -410,7 +410,7 @@ class Http11TestCase(HttpTestCase):
request = Request(self.getURL('largechunkedfile'))
def check(logger):
logger.error.assert_called_once_with(mock.ANY, mock.ANY)
logger.warning.assert_called_once_with(mock.ANY, mock.ANY)
d = self.download_request(request, Spider('foo', download_maxsize=1500))
yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)