diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index 3f29aa323..6efb73abb 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -73,7 +73,7 @@ engine_started Sent when the Scrapy engine has started crawling. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. .. note:: This signal may be fired *after* the :signal:`spider_opened` signal, depending on how the spider was started. So **don't** rely on this signal @@ -88,7 +88,7 @@ engine_stopped Sent when the Scrapy engine is stopped (for example, when a crawling process has finished). - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. item_scraped ------------ @@ -99,7 +99,7 @@ item_scraped Sent when an item has been scraped, after it has passed all the :ref:`topics-item-pipeline` stages (without being dropped). - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param item: the item scraped :type item: dict or :class:`~scrapy.item.Item` object @@ -119,7 +119,7 @@ item_dropped Sent after an item has been dropped from the :ref:`topics-item-pipeline` when some stage raised a :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param item: the item dropped from the :ref:`topics-item-pipeline` :type item: dict or :class:`~scrapy.item.Item` object @@ -144,7 +144,7 @@ item_error Sent when a :ref:`topics-item-pipeline` generates an error (ie. raises an exception), except :exc:`~scrapy.exceptions.DropItem` exception. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param item: the item dropped from the :ref:`topics-item-pipeline` :type item: dict or :class:`~scrapy.item.Item` object @@ -158,6 +158,23 @@ item_error :param failure: the exception raised :type failure: twisted.python.failure.Failure +bytes_received +-------------- + +.. signal:: bytes_received +.. function:: bytes_received(data, request) + + Sent by the HTTP 1.1 download handler when a group of bytes is + received for a specific request. + + This signal does not support returning deferreds from its handlers. + + :param data: the data received by the download handler + :type spider: :class:`bytes` object + + :param request: the request that generated the response + :type request: :class:`~scrapy.http.Request` object + spider_closed ------------- @@ -167,7 +184,7 @@ spider_closed Sent after a spider has been closed. This can be used to release per-spider resources reserved on :signal:`spider_opened`. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param spider: the spider which has been closed :type spider: :class:`~scrapy.spiders.Spider` object @@ -191,7 +208,7 @@ spider_opened reserve per-spider resources, but can be used for any task that needs to be performed when a spider is opened. - This signal supports returning deferreds from their handlers. + This signal supports returning deferreds from its handlers. :param spider: the spider which has been opened :type spider: :class:`~scrapy.spiders.Spider` object @@ -215,7 +232,7 @@ spider_idle You may raise a :exc:`~scrapy.exceptions.DontCloseSpider` exception to prevent the spider from being closed. - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param spider: the spider which has gone idle :type spider: :class:`~scrapy.spiders.Spider` object @@ -234,7 +251,7 @@ spider_error Sent when a spider callback generates an error (ie. raises an exception). - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param failure: the exception raised :type failure: twisted.python.failure.Failure @@ -254,7 +271,7 @@ request_scheduled Sent when the engine schedules a :class:`~scrapy.http.Request`, to be downloaded later. - The signal does not support returning deferreds from their handlers. + The signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler :type request: :class:`~scrapy.http.Request` object @@ -271,7 +288,7 @@ request_dropped Sent when a :class:`~scrapy.http.Request`, scheduled by the engine to be downloaded later, is rejected by the scheduler. - The signal does not support returning deferreds from their handlers. + The signal does not support returning deferreds from its handlers. :param request: the request that reached the scheduler :type request: :class:`~scrapy.http.Request` object @@ -287,7 +304,7 @@ request_reached_downloader Sent when a :class:`~scrapy.http.Request` reached downloader. - The signal does not support returning deferreds from their handlers. + The signal does not support returning deferreds from its handlers. :param request: the request that reached downloader :type request: :class:`~scrapy.http.Request` object @@ -304,7 +321,7 @@ response_received Sent when the engine receives a new :class:`~scrapy.http.Response` from the downloader. - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param response: the response received :type response: :class:`~scrapy.http.Response` object @@ -323,7 +340,7 @@ response_downloaded Sent by the downloader right after a ``HTTPResponse`` is downloaded. - This signal does not support returning deferreds from their handlers. + This signal does not support returning deferreds from its handlers. :param response: the response downloaded :type response: :class:`~scrapy.http.Response` object diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 5a5f6cf0a..92c3d5f5c 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -16,6 +16,7 @@ from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from zope.interface import implementer +from scrapy import signals from scrapy.core.downloader.tls import openssl_methods from scrapy.core.downloader.webclient import _parse from scrapy.exceptions import ScrapyDeprecationWarning @@ -32,6 +33,7 @@ class HTTP11DownloadHandler: lazy = False def __init__(self, settings, crawler=None): + self.crawler = crawler self._pool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False @@ -76,6 +78,7 @@ class HTTP11DownloadHandler: maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), fail_on_dataloss=self._fail_on_dataloss, + crawler=self.crawler, ) return agent.download_request(request) @@ -272,7 +275,7 @@ class ScrapyAgent(object): _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, - maxsize=0, warnsize=0, fail_on_dataloss=True): + maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress @@ -281,6 +284,7 @@ class ScrapyAgent(object): self._warnsize = warnsize self._fail_on_dataloss = fail_on_dataloss self._txresponse = None + self._crawler = crawler def _get_agent(self, request, timeout): bindaddress = request.meta.get('bindaddress') or self._bindAddress @@ -409,7 +413,15 @@ class ScrapyAgent(object): d = defer.Deferred(_cancel) txresponse.deliverBody( - _ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss) + _ResponseReader( + d, + txresponse, + request, + maxsize, + warnsize, + fail_on_dataloss, + self._crawler, + ) ) # save response for timeouts @@ -445,7 +457,7 @@ class _RequestBodyProducer(object): class _ResponseReader(protocol.Protocol): - def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss): + def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler): self._finished = finished self._txresponse = txresponse self._request = request @@ -456,6 +468,7 @@ class _ResponseReader(protocol.Protocol): self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 + self._crawler = crawler def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. @@ -465,6 +478,12 @@ class _ResponseReader(protocol.Protocol): self._bodybuf.write(bodyBytes) self._bytes_received += len(bodyBytes) + self._crawler.signals.send_catch_log( + signal=signals.bytes_received, + data=bodyBytes, + request=self._request, + ) + if self._maxsize and self._bytes_received > self._maxsize: logger.error("Received (%(bytes)s) bytes larger than download " "max size (%(maxsize)s) in request %(request)s.", diff --git a/scrapy/signals.py b/scrapy/signals.py index 6b9125302..590421893 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -16,6 +16,7 @@ request_dropped = object() request_reached_downloader = object() response_received = object() response_downloaded = object() +bytes_received = object() item_scraped = object() item_dropped = object() item_error = object()